UNPKG

@lyleunderwood/streaming-zipper

Version:

Memory-efficient streaming ZIP creation with automatic backpressure control. Supports parallel reading + sequential writing for both Web Streams and Node.js streams with ZIP64 support.

1,558 lines (1,550 loc) 62.7 kB
// src/zip-format.ts var ZIP_CONSTANTS = { // Signatures LOCAL_FILE_HEADER_SIGNATURE: 67324752, CENTRAL_DIRECTORY_SIGNATURE: 33639248, END_OF_CENTRAL_DIRECTORY_SIGNATURE: 101010256, ZIP64_END_OF_CENTRAL_DIRECTORY_SIGNATURE: 101075792, ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE: 117853008, DATA_DESCRIPTOR_SIGNATURE: 134695760, // Compression methods COMPRESSION_STORE: 0, COMPRESSION_DEFLATE: 8, // General purpose bit flags FLAG_ENCRYPTED: 1, FLAG_DATA_DESCRIPTOR: 8, FLAG_UTF8: 2048, // ZIP64 constants ZIP64_LIMIT: 4294967295, ZIP64_LIMIT_16: 65535, ZIP64_EXTRA_FIELD_TYPE: 1, // Version numbers VERSION_MADE_BY: 831, // 3.1 Unix VERSION_NEEDED_EXTRACT: 20, // 2.0 VERSION_NEEDED_EXTRACT_ZIP64: 45 // 4.5 }; function isFastPathStoreEntry(entry) { return typeof entry.crc32 === "number" && typeof entry.size === "number" && (entry.preCompressed === void 0 || entry.preCompressed === false); } function isFastPathDeflateEntry(entry) { return typeof entry.crc32 === "number" && typeof entry.compressedSize === "number" && typeof entry.uncompressedSize === "number" && entry.preCompressed === true; } function canUseFastPath(entry, compressionMethod) { if (compressionMethod === "store") { return isFastPathStoreEntry(entry); } else if (compressionMethod === "deflate") { return isFastPathDeflateEntry(entry); } return false; } function dateToDosDateTime(date) { const year = date.getFullYear(); const month = date.getMonth() + 1; const day = date.getDate(); const hours = date.getHours(); const minutes = date.getMinutes(); const seconds = Math.floor(date.getSeconds() / 2); const dosDate = year - 1980 << 9 | month << 5 | day; const dosTime = hours << 11 | minutes << 5 | seconds; return { date: dosDate, time: dosTime }; } function needsZip64(uncompressedSize, compressedSize, localHeaderOffset, centralDirSize, entryCount) { return Number(uncompressedSize) >= ZIP_CONSTANTS.ZIP64_LIMIT || Number(compressedSize) >= ZIP_CONSTANTS.ZIP64_LIMIT || Number(localHeaderOffset) >= ZIP_CONSTANTS.ZIP64_LIMIT || Number(centralDirSize) >= ZIP_CONSTANTS.ZIP64_LIMIT || entryCount >= ZIP_CONSTANTS.ZIP64_LIMIT_16; } function createZip64ExtraField(uncompressedSize, compressedSize, localHeaderOffset, diskNumber) { const fields = []; if (uncompressedSize !== void 0) fields.push(uncompressedSize); if (compressedSize !== void 0) fields.push(compressedSize); if (localHeaderOffset !== void 0) fields.push(localHeaderOffset); if (diskNumber !== void 0) fields.push(BigInt(diskNumber)); const size = fields.length * 8 + (diskNumber !== void 0 ? -4 : 0); const buffer = new ArrayBuffer(4 + size); const view = new DataView(buffer); view.setUint16(0, ZIP_CONSTANTS.ZIP64_EXTRA_FIELD_TYPE, true); view.setUint16(2, size, true); let offset = 4; for (const field of fields) { if (field === BigInt(diskNumber) && diskNumber !== void 0) { view.setUint32(offset, Number(field), true); offset += 4; } else { view.setBigUint64(offset, field, true); offset += 8; } } return new Uint8Array(buffer); } var UNIX_FILE_TYPES = { REGULAR_FILE: 32768, // S_IFREG DIRECTORY: 16384, // S_IFDIR SYMBOLIC_LINK: 40960 // S_IFLNK }; var DEFAULT_PERMISSIONS = { FILE: 420, // rw-r--r-- DIRECTORY: 493, // rwxr-xr-x EXECUTABLE: 493 // rwxr-xr-x }; function createExternalAttributes(permissions, isDirectory) { let unixPermissions = permissions; if (unixPermissions === void 0) { unixPermissions = isDirectory ? DEFAULT_PERMISSIONS.DIRECTORY : DEFAULT_PERMISSIONS.FILE; } const fileType = isDirectory ? UNIX_FILE_TYPES.DIRECTORY : UNIX_FILE_TYPES.REGULAR_FILE; const unixMode = fileType | unixPermissions; const dosAttributes = isDirectory ? 16 : 32; return (unixMode << 16 | dosAttributes) >>> 0; } // src/zip-serializer.ts function serializeLocalFileHeader(header) { const totalSize = 30 + header.filenameLength + header.extraFieldLength; const buffer = new ArrayBuffer(totalSize); const view = new DataView(buffer); view.setUint32(0, header.signature, true); view.setUint16(4, header.versionNeeded, true); view.setUint16(6, header.flags, true); view.setUint16(8, header.compressionMethod, true); view.setUint16(10, header.lastModTime, true); view.setUint16(12, header.lastModDate, true); view.setUint32(14, header.crc32, true); view.setUint32(18, header.compressedSize, true); view.setUint32(22, header.uncompressedSize, true); view.setUint16(26, header.filenameLength, true); view.setUint16(28, header.extraFieldLength, true); const result = new Uint8Array(buffer); result.set(header.filename, 30); result.set(header.extraField, 30 + header.filenameLength); return result; } function serializeCentralDirectoryHeader(header) { const totalSize = 46 + header.filenameLength + header.extraFieldLength + header.commentLength; const buffer = new ArrayBuffer(totalSize); const view = new DataView(buffer); view.setUint32(0, header.signature, true); view.setUint16(4, header.versionMadeBy, true); view.setUint16(6, header.versionNeeded, true); view.setUint16(8, header.flags, true); view.setUint16(10, header.compressionMethod, true); view.setUint16(12, header.lastModTime, true); view.setUint16(14, header.lastModDate, true); view.setUint32(16, header.crc32, true); view.setUint32(20, header.compressedSize, true); view.setUint32(24, header.uncompressedSize, true); view.setUint16(28, header.filenameLength, true); view.setUint16(30, header.extraFieldLength, true); view.setUint16(32, header.commentLength, true); view.setUint16(34, header.diskNumber, true); view.setUint16(36, header.internalAttributes, true); view.setUint32(38, header.externalAttributes, true); view.setUint32(42, header.localHeaderOffset, true); const result = new Uint8Array(buffer); let offset = 46; result.set(header.filename, offset); offset += header.filenameLength; result.set(header.extraField, offset); offset += header.extraFieldLength; result.set(header.comment, offset); return result; } function serializeEndOfCentralDirectory(eocd) { const totalSize = 22 + eocd.commentLength; const buffer = new ArrayBuffer(totalSize); const view = new DataView(buffer); view.setUint32(0, eocd.signature, true); view.setUint16(4, eocd.diskNumber, true); view.setUint16(6, eocd.centralDirDisk, true); view.setUint16(8, eocd.centralDirRecords, true); view.setUint16(10, eocd.totalRecords, true); view.setUint32(12, eocd.centralDirSize, true); view.setUint32(16, eocd.centralDirOffset, true); view.setUint16(20, eocd.commentLength, true); const result = new Uint8Array(buffer); result.set(eocd.comment, 22); return result; } function serializeZip64EndOfCentralDirectory(eocd) { const buffer = new ArrayBuffer(56); const view = new DataView(buffer); view.setUint32(0, eocd.signature, true); view.setBigUint64(4, eocd.recordSize, true); view.setUint16(12, eocd.versionMadeBy, true); view.setUint16(14, eocd.versionNeeded, true); view.setUint32(16, eocd.diskNumber, true); view.setUint32(20, eocd.centralDirDisk, true); view.setBigUint64(24, eocd.centralDirRecords, true); view.setBigUint64(32, eocd.totalRecords, true); view.setBigUint64(40, eocd.centralDirSize, true); view.setBigUint64(48, eocd.centralDirOffset, true); return new Uint8Array(buffer); } function serializeZip64EndOfCentralDirectoryLocator(locator) { const buffer = new ArrayBuffer(20); const view = new DataView(buffer); view.setUint32(0, locator.signature, true); view.setUint32(4, locator.zip64EndDisk, true); view.setBigUint64(8, locator.zip64EndOffset, true); view.setUint32(16, locator.totalDisks, true); return new Uint8Array(buffer); } function serializeDataDescriptor(descriptor) { const hasSignature = descriptor.signature !== void 0; const size = hasSignature ? 16 : 12; const buffer = new ArrayBuffer(size); const view = new DataView(buffer); let offset = 0; if (hasSignature) { view.setUint32(offset, descriptor.signature, true); offset += 4; } view.setUint32(offset, descriptor.crc32, true); view.setUint32(offset + 4, descriptor.compressedSize, true); view.setUint32(offset + 8, descriptor.uncompressedSize, true); return new Uint8Array(buffer); } function serializeZip64DataDescriptor(descriptor) { const hasSignature = descriptor.signature !== void 0; const size = hasSignature ? 24 : 20; const buffer = new ArrayBuffer(size); const view = new DataView(buffer); let offset = 0; if (hasSignature) { view.setUint32(offset, descriptor.signature, true); offset += 4; } view.setUint32(offset, descriptor.crc32, true); view.setBigUint64(offset + 4, descriptor.compressedSize, true); view.setBigUint64(offset + 12, descriptor.uncompressedSize, true); return new Uint8Array(buffer); } // src/crc32.ts var crcTable = null; function generateCrcTable() { if (crcTable) return crcTable; crcTable = new Uint32Array(256); for (let i = 0; i < 256; i++) { let crc = i; for (let j = 0; j < 8; j++) { if (crc & 1) { crc = crc >>> 1 ^ 3988292384; } else { crc = crc >>> 1; } } crcTable[i] = crc; } return crcTable; } function crc32(data, crc = 0) { const table = generateCrcTable(); crc = crc ^ 4294967295; for (let i = 0; i < data.length; i++) { crc = table[(crc ^ data[i]) & 255] ^ crc >>> 8; } return (crc ^ 4294967295) >>> 0; } var CRC32Stream = class { constructor() { this.crc = 0; this.table = generateCrcTable(); this.crc = 4294967295; } /** * Update CRC32 with new data chunk */ update(data) { for (let i = 0; i < data.length; i++) { this.crc = this.table[(this.crc ^ data[i]) & 255] ^ this.crc >>> 8; } } /** * Get the final CRC32 value */ digest() { return (this.crc ^ 4294967295) >>> 0; } /** * Reset the CRC32 calculator */ reset() { this.crc = 4294967295; } /** * Get current CRC32 value without finalization */ getCurrentValue() { return (this.crc ^ 4294967295) >>> 0; } }; // src/logger.ts import { createLogger } from "@lyleunderwood/isomorphic-logger"; var logger = createLogger({ level: process.env.LOG_LEVEL || "info" }); var logger_default = logger; // src/compression.ts async function compressStore(data) { const crc322 = new CRC32Stream(); crc322.update(data); return { compressedData: data, crc32: crc322.digest(), compressedSize: data.length, uncompressedSize: data.length }; } async function compressDeflate(data) { const crc32Stream = new CRC32Stream(); crc32Stream.update(data); try { const zlib = await import("zlib").catch(() => null); if (zlib) { const compressed = zlib.deflateRawSync(data); return { compressedData: new Uint8Array(compressed), crc32: crc32Stream.digest(), compressedSize: compressed.length, uncompressedSize: data.length }; } if (typeof CompressionStream !== "undefined") { const stream = new CompressionStream("deflate-raw"); const writer = stream.writable.getWriter(); const reader = stream.readable.getReader(); writer.write(data); writer.close(); const chunks = []; let result; while (!(result = await reader.read()).done) { chunks.push(result.value); } const compressed = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0)); let offset = 0; for (const chunk of chunks) { compressed.set(chunk, offset); offset += chunk.length; } return { compressedData: compressed, crc32: crc32Stream.digest(), compressedSize: compressed.length, uncompressedSize: data.length }; } } catch (error) { logger.warn({ error: error.message, stack: error.stack }, "Deflate compression failed, falling back to store method"); } return compressStore(data); } function createCompressionStream(inputStream, method = "deflate") { const crc32Stream = new CRC32Stream(); let compressedSize = 0; let uncompressedSize = 0; let crc32Resolver; let compressedSizeResolver; let uncompressedSizeResolver; const crc32Promise = new Promise((resolve) => { crc32Resolver = resolve; }); const compressedSizePromise = new Promise((resolve) => { compressedSizeResolver = resolve; }); const uncompressedSizePromise = new Promise((resolve) => { uncompressedSizeResolver = resolve; }); const webStream = "getReader" in inputStream ? inputStream : new ReadableStream({ start(controller) { const nodeStream = inputStream; nodeStream.on("data", (chunk) => { const uint8Array = new Uint8Array(chunk); controller.enqueue(uint8Array); }); nodeStream.on("end", () => controller.close()); nodeStream.on("error", (err) => controller.error(err)); } }); if (method === "store") { const storeTransform = new TransformStream({ transform(chunk, controller) { crc32Stream.update(chunk); uncompressedSize += chunk.length; compressedSize += chunk.length; controller.enqueue(chunk); }, flush() { crc32Resolver(crc32Stream.digest()); compressedSizeResolver(compressedSize); uncompressedSizeResolver(uncompressedSize); } }); const readable = webStream.pipeThrough(storeTransform); return { readable, crc32Promise, compressedSizePromise, uncompressedSizePromise }; } else { return createDeflateCompressionStream( webStream, crc32Stream, crc32Resolver, compressedSizeResolver, uncompressedSizeResolver, crc32Promise, compressedSizePromise, uncompressedSizePromise ); } } function createDeflateCompressionStream(webStream, crc32Stream, crc32Resolver, compressedSizeResolver, uncompressedSizeResolver, crc32Promise, compressedSizePromise, uncompressedSizePromise) { let compressedSize = 0; let uncompressedSize = 0; if (typeof process !== "undefined" && process.versions && process.versions.node) { try { const crcTransform = new TransformStream({ transform(chunk, controller) { crc32Stream.update(chunk); uncompressedSize += chunk.length; controller.enqueue(chunk); } }); const sizeCounterTransform = new TransformStream({ transform(chunk, controller) { compressedSize += chunk.length; controller.enqueue(chunk); }, flush() { crc32Resolver(crc32Stream.digest()); compressedSizeResolver(compressedSize); uncompressedSizeResolver(uncompressedSize); } }); const deflateStream = createNodeDeflateStream(); const readable = webStream.pipeThrough(crcTransform).pipeThrough(deflateStream).pipeThrough(sizeCounterTransform); return { readable, crc32Promise, compressedSizePromise, uncompressedSizePromise }; } catch (error) { logger.warn({ error: error.message, stack: error.stack }, "Node.js deflate failed, falling back to browser compression"); } } if (typeof CompressionStream !== "undefined") { try { const crcTransform = new TransformStream({ transform(chunk, controller) { crc32Stream.update(chunk); uncompressedSize += chunk.length; controller.enqueue(chunk); } }); const sizeCounterTransform = new TransformStream({ transform(chunk, controller) { compressedSize += chunk.length; controller.enqueue(chunk); }, flush() { crc32Resolver(crc32Stream.digest()); compressedSizeResolver(compressedSize); uncompressedSizeResolver(uncompressedSize); } }); const deflateStream = new CompressionStream("deflate-raw"); const readable = webStream.pipeThrough(crcTransform).pipeThrough(deflateStream).pipeThrough(sizeCounterTransform); return { readable, crc32Promise, compressedSizePromise, uncompressedSizePromise }; } catch (error) { logger.warn({ error: error.message, stack: error.stack }, "Browser compression failed, falling back to store method"); } } logger.warn("No compression available, using store method"); return createCompressionStream(webStream, "store"); } function createNodeDeflateStream() { let deflate = null; let streamClosed = false; return new TransformStream({ async start(controller) { try { const zlib = await import("zlib"); deflate = zlib.createDeflateRaw(); deflate.on("data", (chunk) => { if (!streamClosed) { controller.enqueue(new Uint8Array(chunk)); } }); deflate.on("end", () => { if (!streamClosed) { streamClosed = true; controller.terminate(); } }); deflate.on("error", (err) => { if (!streamClosed) { streamClosed = true; controller.error(err); } }); } catch (error) { controller.error(error); } }, transform(chunk, controller) { if (deflate && !streamClosed) { deflate.write(Buffer.from(chunk)); } else if (!deflate) { controller.error(new Error("Deflate stream not initialized")); } }, flush(controller) { return new Promise((resolve, reject) => { if (!deflate || streamClosed) { resolve(); return; } const onEnd = () => { deflate.removeListener("error", onError); if (!streamClosed) { streamClosed = true; controller.terminate(); } resolve(); }; const onError = (err) => { deflate.removeListener("end", onEnd); if (!streamClosed) { streamClosed = true; controller.error(err); } reject(err); }; deflate.once("end", onEnd); deflate.once("error", onError); deflate.end(); }); } }); } // src/entry-buffer.ts var EntryBuffer = class { constructor(entry, index, compressionMethod = "store", maxBufferSize = 1024 * 1024) { this.entry = entry; this.index = index; this.compressionMethod = compressionMethod; this.chunks = []; this.totalSize = 0; this.crc32Stream = new CRC32Stream(); this.state = "pending" /* PENDING */; this.readPromise = null; this.readResolver = null; this.readError = null; this.metadata = {}; // Backpressure coordination this.waitingReaders = []; this.maxBufferSize = maxBufferSize; } /** * Get current entry state */ getState() { return this.state; } /** * Check if entry is ready to be written */ isReady() { return this.state === "ready" /* READY */; } /** * Check if entry has been completely read */ isReadComplete() { return this.state === "ready" /* READY */ || this.state === "writing" /* WRITING */ || this.state === "completed" /* COMPLETED */ || this.state === "error" /* ERROR */; } /** * Get buffered size in bytes */ getBufferedSize() { const actualSize = this.chunks.reduce((total, chunk) => total + chunk.data.length, 0); if (actualSize !== this.totalSize) { logger_default.debug({ actualSize, totalSize: this.totalSize }, "Buffer size mismatch detected, correcting"); this.totalSize = actualSize; } return this.totalSize; } /** * Check if buffer has space for more data */ hasBufferSpace() { return this.totalSize < this.maxBufferSize; } /** * Start reading from the entry's data stream */ async startReading() { if (this.state !== "pending" /* PENDING */) { throw new Error(`Cannot start reading entry in state: ${this.state}`); } this.state = "reading" /* READING */; this.readPromise = this.performRead(); return this.readPromise; } /** * Wait for entry to be ready for writing */ async waitForReady() { if (this.state === "ready" /* READY */) { return; } if (this.state === "error" /* ERROR */) { throw this.readError || new Error("Entry reading failed"); } if (this.readPromise) { await this.readPromise; } const currentState = this.state; switch (currentState) { case "error" /* ERROR */: throw this.readError || new Error("Entry reading failed"); case "ready" /* READY */: return; default: throw new Error(`Unexpected state after reading: ${currentState}`); } } /** * Read the next chunk from the buffer */ readChunk() { if (this.chunks.length === 0) { return null; } const chunk = this.chunks.shift(); if (chunk) { this.totalSize -= chunk.data.length; } return chunk || null; } /** * Get entry metadata */ getMetadata() { if (!this.isReadComplete()) { throw new Error("Metadata not available until reading is complete"); } return { crc32: this.metadata.crc32, compressedSize: this.metadata.compressedSize, uncompressedSize: this.metadata.uncompressedSize, localHeaderOffset: this.metadata.localHeaderOffset || 0, compressionMethod: this.compressionMethod }; } /** * Set the local header offset */ setLocalHeaderOffset(offset) { this.metadata.localHeaderOffset = offset; } /** * Mark entry as being written */ startWriting() { if (this.state !== "ready" /* READY */) { throw new Error(`Cannot start writing entry in state: ${this.state}`); } this.state = "writing" /* WRITING */; } /** * Mark entry as completed */ markCompleted() { if (this.state !== "writing" /* WRITING */) { throw new Error(`Cannot mark entry completed from state: ${this.state}`); } this.state = "completed" /* COMPLETED */; } /** * Get total uncompressed size */ getUncompressedSize() { return this.metadata.uncompressedSize || 0; } /** * Get total compressed size (same as uncompressed for store method) */ getCompressedSize() { return this.metadata.compressedSize || 0; } /** * Get CRC32 checksum */ getCRC32() { return this.metadata.crc32 || 0; } /** * Perform the actual reading from the stream */ async performRead() { logger_default.debug({ entryName: this.entry.name }, "Starting to read entry data"); try { logger_default.debug({ entryName: this.entry.name }, "Reading entry data"); const inputStream = "getReader" in this.entry.data ? this.entry.data : this.createWebStreamFromNodeStream(this.entry.data); logger_default.debug("Starting to read entry chunks"); const compressionResult = createCompressionStream(inputStream, this.compressionMethod); const reader = compressionResult.readable.getReader(); logger_default.debug("Entry chunk reader created"); let chunkOffset = 0; try { while (true) { logger_default.debug("Reading entry chunk"); const { done, value } = await reader.read(); logger_default.debug({ done }, "Entry chunk read"); if (done) break; if (!this.hasBufferSpace()) { logger_default.debug("Waiting for buffer space"); await this.waitForBufferSpace(); logger_default.debug("Buffer space available"); } this.chunks.push({ data: value, offset: chunkOffset }); this.totalSize += value.length; chunkOffset += value.length; } } finally { logger_default.debug("Releasing entry chunk reader"); reader.releaseLock(); } const [crc322, compressedSize, uncompressedSize] = await Promise.all([ compressionResult.crc32Promise, compressionResult.compressedSizePromise, compressionResult.uncompressedSizePromise ]); this.metadata.crc32 = crc322; this.metadata.compressedSize = compressedSize; this.metadata.uncompressedSize = uncompressedSize; this.state = "ready" /* READY */; logger_default.debug({ crc32: crc322, compressedSize, uncompressedSize }, "Entry read complete"); if (this.readResolver) { logger_default.debug("Calling read resolver"); this.readResolver(); } } catch (error) { this.readError = error instanceof Error ? error : new Error(String(error)); this.state = "error" /* ERROR */; if (this.readResolver) { this.readResolver(); } } } /** * Wait for buffer space to become available */ async waitForBufferSpace() { if (this.hasBufferSpace()) { return; } return new Promise((resolve) => { if (this.hasBufferSpace()) { resolve(); return; } this.waitingReaders.push(resolve); }); } /** * Notify that buffer space has been freed up (called when chunks are consumed) */ notifySpaceFreed() { if (this.waitingReaders.length > 0) { const hasSpace = this.hasBufferSpace(); if (hasSpace) { const resolvers = this.waitingReaders.splice(0); logger_default.debug({ bufferedSize: this.getBufferedSize(), maxSize: this.maxBufferSize, waitingReaders: resolvers.length }, "Notifying waiting readers of available buffer space"); resolvers.forEach((resolve) => { try { resolve(); } catch (error) { logger_default.error({ error }, "Error resolving waiting reader"); } }); } } } /** * Create a Web ReadableStream from a Node.js stream */ createWebStreamFromNodeStream(nodeStream) { logger_default.debug("Converting Node.js stream to Web Stream"); return new ReadableStream({ start(controller) { nodeStream.on("data", (chunk) => { const uint8Array = new Uint8Array(chunk); controller.enqueue(uint8Array); }); nodeStream.on("end", () => { controller.close(); }); nodeStream.on("error", (err) => { controller.error(err); }); }, cancel() { if ("destroy" in nodeStream && typeof nodeStream.destroy === "function") { nodeStream.destroy(); } } }); } }; // src/parallel-reader.ts var ParallelReader = class { constructor(options = {}) { this.entryBuffers = []; this.readingPromises = /* @__PURE__ */ new Map(); this.activeReads = 0; this.options = { maxBufferSize: 1024 * 1024, // 1MB per entry maxConcurrentReads: 10, // Max concurrent reads compression: "store", // Default compression ...options }; } /** * Add an entry to be read in parallel */ addEntry(entry) { const index = this.entryBuffers.length; const entryBuffer = new EntryBuffer( entry, index, this.options.compression, this.options.maxBufferSize ); this.entryBuffers.push(entryBuffer); this.startReadingIfPossible(entryBuffer); return entryBuffer; } /** * Get all entry buffers */ getEntryBuffers() { return [...this.entryBuffers]; } /** * Get entry buffer by index */ getEntryBuffer(index) { return this.entryBuffers[index]; } /** * Get the next ready entry in order */ getNextReadyEntry() { for (const buffer of this.entryBuffers) { if (buffer.getState() === "ready" /* READY */) { return buffer; } if (buffer.getState() !== "completed" /* COMPLETED */) { return null; } } return null; } /** * Check if there are any entries still being read */ hasActiveReads() { return this.activeReads > 0 || this.readingPromises.size > 0; } /** * Wait for the next entry to become ready */ async waitForNextReady() { const nextReady = this.getNextReadyEntry(); if (nextReady) { return nextReady; } let nextEntry = null; for (const buffer of this.entryBuffers) { if (buffer.getState() === "completed" /* COMPLETED */) { continue; } nextEntry = buffer; break; } if (!nextEntry) { return null; } if (nextEntry.getState() === "pending" /* PENDING */) { this.startReadingIfPossible(nextEntry); } if (nextEntry.getState() !== "ready" /* READY */) { await nextEntry.waitForReady(); } return nextEntry.isReady() ? nextEntry : null; } /** * Wait for all entries to complete reading */ async waitForAllComplete() { for (const buffer of this.entryBuffers) { if (buffer.getState() === "pending" /* PENDING */) { logger_default.debug({ entryName: buffer.entry.name }, "Starting to read pending entry"); this.startReadingIfPossible(buffer); } } const promises = Array.from(this.readingPromises.values()); if (promises.length > 0) { logger_default.debug({ count: promises.length }, "Waiting for all reading promises to complete"); await Promise.all(promises); } } /** * Get statistics about reading progress */ getStats() { const stats = { total: this.entryBuffers.length, pending: 0, reading: 0, ready: 0, writing: 0, completed: 0, errors: 0 }; for (const buffer of this.entryBuffers) { switch (buffer.getState()) { case "pending" /* PENDING */: stats.pending++; break; case "reading" /* READING */: stats.reading++; break; case "ready" /* READY */: stats.ready++; break; case "writing" /* WRITING */: stats.writing++; break; case "completed" /* COMPLETED */: stats.completed++; break; case "error" /* ERROR */: stats.errors++; break; } } return stats; } /** * Start reading an entry if we have capacity */ startReadingIfPossible(entryBuffer) { if (entryBuffer.getState() !== "pending" /* PENDING */) { return; } if (this.activeReads >= this.options.maxConcurrentReads) { return; } this.activeReads++; logger_default.debug({ entryName: entryBuffer.entry.name }, "Starting to read entry"); const readPromise = entryBuffer.startReading().finally(() => { logger_default.debug({ entryName: entryBuffer.entry.name }, "Completed reading entry"); this.activeReads--; this.readingPromises.delete(entryBuffer.index); this.startNextPendingEntry(); }); this.readingPromises.set(entryBuffer.index, readPromise); } /** * Try to start reading the next pending entry */ startNextPendingEntry() { if (this.activeReads >= this.options.maxConcurrentReads) { return; } for (const buffer of this.entryBuffers) { if (buffer.getState() === "pending" /* PENDING */) { this.startReadingIfPossible(buffer); break; } } } }; // src/write-queue.ts var WriteQueue = class { constructor(options = {}) { this.outputController = null; this.currentOffset = 0; this.centralDirectoryEntries = []; this.writeInProgress = false; this.options = { compression: "store", ...options }; logger.debug({ options: this.options }, "WriteQueue initialized"); } /** * Set the output controller for streaming data */ setOutputController(controller) { this.outputController = controller; } /** * Write an entry buffer to the output stream */ async writeEntry(entryBuffer) { if (this.writeInProgress) { throw new Error("Another write operation is already in progress"); } if (!entryBuffer.isReady()) { throw new Error("Entry buffer is not ready for writing"); } const entryName = entryBuffer.entry.name; logger.debug({ entryName, currentOffset: this.currentOffset }, "Starting to write buffered entry"); this.writeInProgress = true; try { entryBuffer.startWriting(); await this.performEntryWrite(entryBuffer); entryBuffer.markCompleted(); logger.debug({ entryName, finalOffset: this.currentOffset }, "Completed writing buffered entry"); } catch (error) { logger.error({ entryName, error: error.message, stack: error.stack }, "Error writing buffered entry"); throw error; } finally { this.writeInProgress = false; } } /** * Write a direct stream entry immediately to the output stream */ async writeDirectStreamEntry(directEntry) { if (this.writeInProgress) { throw new Error("Another write operation is already in progress"); } if (!directEntry.isReadyForStreaming()) { throw new Error("Direct stream entry is not ready for streaming"); } const entryName = directEntry.entry.name; logger.debug({ entryName, currentOffset: this.currentOffset }, "Starting to write direct stream entry"); this.writeInProgress = true; try { await this.performDirectStreamWrite(directEntry); directEntry.markCompleted(); logger.debug({ entryName, finalOffset: this.currentOffset }, "Completed writing direct stream entry"); } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); logger.error({ entryName, error: err.message, stack: err.stack }, "Error writing direct stream entry"); directEntry.markError(err); throw err; } finally { this.writeInProgress = false; } } /** * Write the central directory to the output stream */ async writeCentralDirectory() { const centralDirStart = this.currentOffset; const entryCount = this.centralDirectoryEntries.length; logger.debug({ entryCount, centralDirStart }, "Writing central directory"); for (let i = 0; i < this.centralDirectoryEntries.length; i++) { const entry = this.centralDirectoryEntries[i]; logger.debug({ entryIndex: i, entryName: entry?.filename }, "Writing central directory entry"); if (!entry) { throw new Error("Central directory entry is missing"); } const serialized = serializeCentralDirectoryHeader(entry); await this.writeToOutput(serialized); } const centralDirSize = this.currentOffset - centralDirStart; logger.debug({ centralDirStart, centralDirSize }, "Central directory written successfully"); return { centralDirOffset: centralDirStart, centralDirSize }; } /** * Get the current offset in the output stream */ getCurrentOffset() { return this.currentOffset; } /** * Get the number of entries written */ getEntryCount() { return this.centralDirectoryEntries.length; } /** * Check if any write operations are in progress */ isWriteInProgress() { return this.writeInProgress; } /** * Perform immediate direct stream write operation */ async performDirectStreamWrite(directEntry) { directEntry.setLocalHeaderOffset(this.currentOffset); const localHeader = this.createDirectStreamLocalFileHeader(directEntry); const headerSize = serializeLocalFileHeader(localHeader).length; logger.debug({ entryName: directEntry.entry.name, headerSize }, "Writing direct stream local file header"); await this.writeToOutput(serializeLocalFileHeader(localHeader)); const dataStream = directEntry.startDirectStreaming(); const reader = dataStream.getReader(); let streamedDataSize = 0; try { while (true) { const { done, value } = await reader.read(); if (done) { break; } streamedDataSize += value.length; await this.writeToOutput(value); } } finally { reader.releaseLock(); } logger.debug({ entryName: directEntry.entry.name, streamedDataSize }, "Direct stream data written"); const dataDescriptor = this.createDirectStreamDataDescriptor(directEntry); logger.debug({ entryName: directEntry.entry.name, descriptorSize: dataDescriptor.length }, "Writing direct stream data descriptor"); await this.writeToOutput(dataDescriptor); const centralDirEntry = this.createDirectStreamCentralDirectoryHeader(directEntry); this.centralDirectoryEntries.push(centralDirEntry); } /** * Perform the actual entry write operation */ async performEntryWrite(entryBuffer) { entryBuffer.setLocalHeaderOffset(this.currentOffset); const localHeader = this.createLocalFileHeader(entryBuffer); logger.debug({ entryName: entryBuffer.entry.name, headerSize: serializeLocalFileHeader(localHeader).length }, "Writing local file header"); await this.writeToOutput(serializeLocalFileHeader(localHeader)); let chunk; let dataSize = 0; while ((chunk = entryBuffer.readChunk()) !== null) { dataSize += chunk.data.length; await this.writeToOutput(chunk.data); entryBuffer.notifySpaceFreed(); } logger.debug({ entryName: entryBuffer.entry.name, dataSize }, "Entry data written"); if (localHeader.flags & ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR) { const dataDescriptor = this.createDataDescriptor(entryBuffer); logger.debug({ entryName: entryBuffer.entry.name, descriptorSize: dataDescriptor.length }, "Writing data descriptor"); await this.writeToOutput(dataDescriptor); } const centralDirEntry = this.createCentralDirectoryHeader(entryBuffer); this.centralDirectoryEntries.push(centralDirEntry); } /** * Create local file header for entry */ createLocalFileHeader(entryBuffer) { const entry = entryBuffer.entry; const filename = new TextEncoder().encode(entry.name); const lastModified = entry.lastModified || /* @__PURE__ */ new Date(); const { date, time } = dateToDosDateTime(lastModified); const flags = ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8; const extraField = new Uint8Array(0); return { signature: ZIP_CONSTANTS.LOCAL_FILE_HEADER_SIGNATURE, versionNeeded: ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT, flags, compressionMethod: entryBuffer.getMetadata().compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE, lastModTime: time, lastModDate: date, crc32: 0, // Will be in data descriptor compressedSize: 0, // Will be in data descriptor uncompressedSize: 0, // Will be in data descriptor filenameLength: filename.length, extraFieldLength: extraField.length, filename, extraField }; } /** * Create data descriptor for entry */ createDataDescriptor(entryBuffer) { const metadata = entryBuffer.getMetadata(); const needsZip64Format = needsZip64( metadata.uncompressedSize, metadata.compressedSize, 0, 0, 0 ); if (needsZip64Format) { const descriptor = { signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE, crc32: metadata.crc32, compressedSize: BigInt(metadata.compressedSize), uncompressedSize: BigInt(metadata.uncompressedSize) }; return serializeZip64DataDescriptor(descriptor); } else { const descriptor = { signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE, crc32: metadata.crc32, compressedSize: metadata.compressedSize, uncompressedSize: metadata.uncompressedSize }; return serializeDataDescriptor(descriptor); } } /** * Create central directory header for entry */ createCentralDirectoryHeader(entryBuffer) { const entry = entryBuffer.entry; const metadata = entryBuffer.getMetadata(); const filename = new TextEncoder().encode(entry.name); const comment = new TextEncoder().encode(entry.comment || ""); const lastModified = entry.lastModified || /* @__PURE__ */ new Date(); const { date, time } = dateToDosDateTime(lastModified); const needsZip64Format = needsZip64( metadata.uncompressedSize, metadata.compressedSize, metadata.localHeaderOffset, 0, 0 ); let extraField = new Uint8Array(0); let compressedSize = metadata.compressedSize; let uncompressedSize = metadata.uncompressedSize; let localHeaderOffset = metadata.localHeaderOffset; if (needsZip64Format) { extraField = createZip64ExtraField( BigInt(metadata.uncompressedSize), BigInt(metadata.compressedSize), BigInt(metadata.localHeaderOffset) ); compressedSize = ZIP_CONSTANTS.ZIP64_LIMIT; uncompressedSize = ZIP_CONSTANTS.ZIP64_LIMIT; localHeaderOffset = ZIP_CONSTANTS.ZIP64_LIMIT; } return { signature: ZIP_CONSTANTS.CENTRAL_DIRECTORY_SIGNATURE, versionMadeBy: ZIP_CONSTANTS.VERSION_MADE_BY, versionNeeded: needsZip64Format ? ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT_ZIP64 : ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT, flags: ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8, compressionMethod: metadata.compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE, lastModTime: time, lastModDate: date, crc32: metadata.crc32, compressedSize, uncompressedSize, filenameLength: filename.length, extraFieldLength: extraField.length, commentLength: comment.length, diskNumber: 0, internalAttributes: 0, externalAttributes: createExternalAttributes(entry.permissions, entry.name.endsWith("/")), localHeaderOffset, filename, extraField, comment }; } /** * Create local file header for direct stream entry */ createDirectStreamLocalFileHeader(directEntry) { const entry = directEntry.entry; const metadata = directEntry.getMetadata(); const filename = new TextEncoder().encode(entry.name); const lastModified = entry.lastModified || /* @__PURE__ */ new Date(); const { date, time } = dateToDosDateTime(lastModified); const flags = ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8; const extraField = new Uint8Array(0); return { signature: ZIP_CONSTANTS.LOCAL_FILE_HEADER_SIGNATURE, versionNeeded: ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT, flags, compressionMethod: metadata.compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE, lastModTime: time, lastModDate: date, crc32: 0, // Will be in data descriptor compressedSize: 0, // Will be in data descriptor uncompressedSize: 0, // Will be in data descriptor filenameLength: filename.length, extraFieldLength: extraField.length, filename, extraField }; } /** * Create data descriptor for direct stream entry */ createDirectStreamDataDescriptor(directEntry) { const metadata = directEntry.getMetadata(); const needsZip64Format = needsZip64( metadata.uncompressedSize, metadata.compressedSize, 0, 0, 0 ); if (needsZip64Format) { const descriptor = { signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE, crc32: metadata.crc32, compressedSize: BigInt(metadata.compressedSize), uncompressedSize: BigInt(metadata.uncompressedSize) }; return serializeZip64DataDescriptor(descriptor); } else { const descriptor = { signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE, crc32: metadata.crc32, compressedSize: metadata.compressedSize, uncompressedSize: metadata.uncompressedSize }; return serializeDataDescriptor(descriptor); } } /** * Create central directory header for direct stream entry */ createDirectStreamCentralDirectoryHeader(directEntry) { const entry = directEntry.entry; const metadata = directEntry.getMetadata(); const filename = new TextEncoder().encode(entry.name); const comment = new TextEncoder().encode(entry.comment || ""); const lastModified = entry.lastModified || /* @__PURE__ */ new Date(); const { date, time } = dateToDosDateTime(lastModified); const needsZip64Format = needsZip64( metadata.uncompressedSize, metadata.compressedSize, metadata.localHeaderOffset, 0, 0 ); let extraField = new Uint8Array(0); let compressedSize = metadata.compressedSize; let uncompressedSize = metadata.uncompressedSize; let localHeaderOffset = metadata.localHeaderOffset; if (needsZip64Format) { extraField = createZip64ExtraField( BigInt(metadata.uncompressedSize), BigInt(metadata.compressedSize), BigInt(metadata.localHeaderOffset) ); compressedSize = ZIP_CONSTANTS.ZIP64_LIMIT; uncompressedSize = ZIP_CONSTANTS.ZIP64_LIMIT; localHeaderOffset = ZIP_CONSTANTS.ZIP64_LIMIT; } return { signature: ZIP_CONSTANTS.CENTRAL_DIRECTORY_SIGNATURE, versionMadeBy: ZIP_CONSTANTS.VERSION_MADE_BY, versionNeeded: needsZip64Format ? ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT_ZIP64 : ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT, flags: ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8, compressionMethod: metadata.compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE, lastModTime: time, lastModDate: date, crc32: metadata.crc32, compressedSize, uncompressedSize, filenameLength: filename.length, extraFieldLength: extraField.length, commentLength: comment.length, diskNumber: 0, internalAttributes: 0, externalAttributes: createExternalAttributes(entry.permissions, entry.name.endsWith("/")), localHeaderOffset, filename, extraField, comment }; } /** * Write data to the output stream */ async writeToOutput(data) { if (this.outputController) { this.outputController.enqueue(data); this.currentOffset += data.length; } } }; // src/direct-stream-entry.ts var DirectStreamEntry = class { constructor(entry, index, compressionMethod) { this.entry = entry; this.index = index; this.state = "pending" /* PENDING */; this.streamReader = null; this.error = null; if (entry.preCompressed) { this.metadata = { crc32: entry.crc32, compressedSize: entry.compressedSize, uncompressedSize: entry.uncompressedSize, localHeaderOffset: 0, compressionMethod: "deflate" }; } else { const size = entry.size || 0; this.metadata = { crc32: entry.crc32, compressedSize: size, uncompressedSize: size, localHeaderOffset: 0, compressionMethod: "store" }; } } /** * Get current state */ getState() { return this.state; } /** * Check if entry is ready for immediate streaming */ isReadyForStreaming() { return this.state === "pending" /* PENDING */; } /** * Check if streaming is complete */ isCompleted() { return this.state === "completed" /* COMPLETED */; } /** * Check if there was an error */ hasError() { return this.state === "error" /* ERROR */; } /** * Get error if any */ getError() { return this.error; } /** * Get pre-calculated metadata */ getMetadata() { return { ...this.metadata }; } /** * Set local header offset (called during writing) */ setLocalHeaderOffset(offset) { this.metadata.localHeaderOffset = offset; } /** * Start streaming data directly from the source * Returns a readable stream that can be piped directly to output */ startDirectStreaming() { if (this.state !== "pending" /* PENDING */) { throw new Error(`Cannot start streaming entry in state: ${this.state}`); } this.state = "streaming" /* STREAMING */; const inputStream = this.convertToWebStream(this.entry.data); return new ReadableStream({ start: (controller) => { this.streamReader = inputStream.getReader(); this.pumpStream(controller); }, cancel: () => { this.cleanup(); } }); } /** * Mark entry as completed */ markCompleted() { this.state = "completed" /* COMPLETED */; this.cleanup(); } /** * Mark entry as failed */ markError(error) { this.error = error; this.state = "error" /* ERROR */; this.cleanup(); } /** * Pump data from input stream to output controller */ async pumpStream(controller) { try { if (!this.streamReader) { throw new Error("Stream reader not initialized"); } let bytesRead = 0; const expectedSize = this.metadata.compressedSize; while (true) { const { done, value } = await this.streamReader.read(); if (done) { break; } bytesRead += value.length; if (bytesRead > expectedSize) { throw new Error(`Stream exceeded expected size: ${bytesRead} > ${expectedSize}`); } controller.enqueue(value); } if (bytesRead !== expectedSize) { throw new Error(`Stream size mismatch: expected ${expectedSize}, got ${bytesRead}`); } controller.close(); this.markCompleted(); } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); controller.error(err); this.markError(err); } } /** * Convert Node.js stream or ReadableStream to Web ReadableStream */ convertToWebStream(stream) { if ("getReader" in stream) { return stream; } c