@lyleunderwood/streaming-zipper
Version:
Memory-efficient streaming ZIP creation with automatic backpressure control. Supports parallel reading + sequential writing for both Web Streams and Node.js streams with ZIP64 support.
1,522 lines (1,512 loc) • 65.1 kB
JavaScript
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var src_exports = {};
__export(src_exports, {
CRC32Stream: () => CRC32Stream,
DEFAULT_PERMISSIONS: () => DEFAULT_PERMISSIONS,
StreamingZipWriter: () => StreamingZipWriter,
UNIX_FILE_TYPES: () => UNIX_FILE_TYPES,
ZIP_CONSTANTS: () => ZIP_CONSTANTS,
canUseFastPath: () => canUseFastPath,
compressDeflate: () => compressDeflate,
compressStore: () => compressStore,
crc32: () => crc32,
createByteCounterStream: () => createByteCounterStream,
createExternalAttributes: () => createExternalAttributes,
createPassThroughStream: () => createPassThroughStream,
default: () => StreamingZipWriter,
isFastPathDeflateEntry: () => isFastPathDeflateEntry,
isFastPathStoreEntry: () => isFastPathStoreEntry,
nodeStreamToWebStream: () => nodeStreamToWebStream,
streamToUint8Array: () => streamToUint8Array,
uint8ArrayToStream: () => uint8ArrayToStream
});
module.exports = __toCommonJS(src_exports);
// src/zip-format.ts
var ZIP_CONSTANTS = {
// Signatures
LOCAL_FILE_HEADER_SIGNATURE: 67324752,
CENTRAL_DIRECTORY_SIGNATURE: 33639248,
END_OF_CENTRAL_DIRECTORY_SIGNATURE: 101010256,
ZIP64_END_OF_CENTRAL_DIRECTORY_SIGNATURE: 101075792,
ZIP64_END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE: 117853008,
DATA_DESCRIPTOR_SIGNATURE: 134695760,
// Compression methods
COMPRESSION_STORE: 0,
COMPRESSION_DEFLATE: 8,
// General purpose bit flags
FLAG_ENCRYPTED: 1,
FLAG_DATA_DESCRIPTOR: 8,
FLAG_UTF8: 2048,
// ZIP64 constants
ZIP64_LIMIT: 4294967295,
ZIP64_LIMIT_16: 65535,
ZIP64_EXTRA_FIELD_TYPE: 1,
// Version numbers
VERSION_MADE_BY: 831,
// 3.1 Unix
VERSION_NEEDED_EXTRACT: 20,
// 2.0
VERSION_NEEDED_EXTRACT_ZIP64: 45
// 4.5
};
function isFastPathStoreEntry(entry) {
return typeof entry.crc32 === "number" && typeof entry.size === "number" && (entry.preCompressed === void 0 || entry.preCompressed === false);
}
function isFastPathDeflateEntry(entry) {
return typeof entry.crc32 === "number" && typeof entry.compressedSize === "number" && typeof entry.uncompressedSize === "number" && entry.preCompressed === true;
}
function canUseFastPath(entry, compressionMethod) {
if (compressionMethod === "store") {
return isFastPathStoreEntry(entry);
} else if (compressionMethod === "deflate") {
return isFastPathDeflateEntry(entry);
}
return false;
}
function dateToDosDateTime(date) {
const year = date.getFullYear();
const month = date.getMonth() + 1;
const day = date.getDate();
const hours = date.getHours();
const minutes = date.getMinutes();
const seconds = Math.floor(date.getSeconds() / 2);
const dosDate = year - 1980 << 9 | month << 5 | day;
const dosTime = hours << 11 | minutes << 5 | seconds;
return { date: dosDate, time: dosTime };
}
function needsZip64(uncompressedSize, compressedSize, localHeaderOffset, centralDirSize, entryCount) {
return Number(uncompressedSize) >= ZIP_CONSTANTS.ZIP64_LIMIT || Number(compressedSize) >= ZIP_CONSTANTS.ZIP64_LIMIT || Number(localHeaderOffset) >= ZIP_CONSTANTS.ZIP64_LIMIT || Number(centralDirSize) >= ZIP_CONSTANTS.ZIP64_LIMIT || entryCount >= ZIP_CONSTANTS.ZIP64_LIMIT_16;
}
function createZip64ExtraField(uncompressedSize, compressedSize, localHeaderOffset, diskNumber) {
const fields = [];
if (uncompressedSize !== void 0)
fields.push(uncompressedSize);
if (compressedSize !== void 0)
fields.push(compressedSize);
if (localHeaderOffset !== void 0)
fields.push(localHeaderOffset);
if (diskNumber !== void 0)
fields.push(BigInt(diskNumber));
const size = fields.length * 8 + (diskNumber !== void 0 ? -4 : 0);
const buffer = new ArrayBuffer(4 + size);
const view = new DataView(buffer);
view.setUint16(0, ZIP_CONSTANTS.ZIP64_EXTRA_FIELD_TYPE, true);
view.setUint16(2, size, true);
let offset = 4;
for (const field of fields) {
if (field === BigInt(diskNumber) && diskNumber !== void 0) {
view.setUint32(offset, Number(field), true);
offset += 4;
} else {
view.setBigUint64(offset, field, true);
offset += 8;
}
}
return new Uint8Array(buffer);
}
var UNIX_FILE_TYPES = {
REGULAR_FILE: 32768,
// S_IFREG
DIRECTORY: 16384,
// S_IFDIR
SYMBOLIC_LINK: 40960
// S_IFLNK
};
var DEFAULT_PERMISSIONS = {
FILE: 420,
// rw-r--r--
DIRECTORY: 493,
// rwxr-xr-x
EXECUTABLE: 493
// rwxr-xr-x
};
function createExternalAttributes(permissions, isDirectory) {
let unixPermissions = permissions;
if (unixPermissions === void 0) {
unixPermissions = isDirectory ? DEFAULT_PERMISSIONS.DIRECTORY : DEFAULT_PERMISSIONS.FILE;
}
const fileType = isDirectory ? UNIX_FILE_TYPES.DIRECTORY : UNIX_FILE_TYPES.REGULAR_FILE;
const unixMode = fileType | unixPermissions;
const dosAttributes = isDirectory ? 16 : 32;
return (unixMode << 16 | dosAttributes) >>> 0;
}
// src/zip-serializer.ts
function serializeLocalFileHeader(header) {
const totalSize = 30 + header.filenameLength + header.extraFieldLength;
const buffer = new ArrayBuffer(totalSize);
const view = new DataView(buffer);
view.setUint32(0, header.signature, true);
view.setUint16(4, header.versionNeeded, true);
view.setUint16(6, header.flags, true);
view.setUint16(8, header.compressionMethod, true);
view.setUint16(10, header.lastModTime, true);
view.setUint16(12, header.lastModDate, true);
view.setUint32(14, header.crc32, true);
view.setUint32(18, header.compressedSize, true);
view.setUint32(22, header.uncompressedSize, true);
view.setUint16(26, header.filenameLength, true);
view.setUint16(28, header.extraFieldLength, true);
const result = new Uint8Array(buffer);
result.set(header.filename, 30);
result.set(header.extraField, 30 + header.filenameLength);
return result;
}
function serializeCentralDirectoryHeader(header) {
const totalSize = 46 + header.filenameLength + header.extraFieldLength + header.commentLength;
const buffer = new ArrayBuffer(totalSize);
const view = new DataView(buffer);
view.setUint32(0, header.signature, true);
view.setUint16(4, header.versionMadeBy, true);
view.setUint16(6, header.versionNeeded, true);
view.setUint16(8, header.flags, true);
view.setUint16(10, header.compressionMethod, true);
view.setUint16(12, header.lastModTime, true);
view.setUint16(14, header.lastModDate, true);
view.setUint32(16, header.crc32, true);
view.setUint32(20, header.compressedSize, true);
view.setUint32(24, header.uncompressedSize, true);
view.setUint16(28, header.filenameLength, true);
view.setUint16(30, header.extraFieldLength, true);
view.setUint16(32, header.commentLength, true);
view.setUint16(34, header.diskNumber, true);
view.setUint16(36, header.internalAttributes, true);
view.setUint32(38, header.externalAttributes, true);
view.setUint32(42, header.localHeaderOffset, true);
const result = new Uint8Array(buffer);
let offset = 46;
result.set(header.filename, offset);
offset += header.filenameLength;
result.set(header.extraField, offset);
offset += header.extraFieldLength;
result.set(header.comment, offset);
return result;
}
function serializeEndOfCentralDirectory(eocd) {
const totalSize = 22 + eocd.commentLength;
const buffer = new ArrayBuffer(totalSize);
const view = new DataView(buffer);
view.setUint32(0, eocd.signature, true);
view.setUint16(4, eocd.diskNumber, true);
view.setUint16(6, eocd.centralDirDisk, true);
view.setUint16(8, eocd.centralDirRecords, true);
view.setUint16(10, eocd.totalRecords, true);
view.setUint32(12, eocd.centralDirSize, true);
view.setUint32(16, eocd.centralDirOffset, true);
view.setUint16(20, eocd.commentLength, true);
const result = new Uint8Array(buffer);
result.set(eocd.comment, 22);
return result;
}
function serializeZip64EndOfCentralDirectory(eocd) {
const buffer = new ArrayBuffer(56);
const view = new DataView(buffer);
view.setUint32(0, eocd.signature, true);
view.setBigUint64(4, eocd.recordSize, true);
view.setUint16(12, eocd.versionMadeBy, true);
view.setUint16(14, eocd.versionNeeded, true);
view.setUint32(16, eocd.diskNumber, true);
view.setUint32(20, eocd.centralDirDisk, true);
view.setBigUint64(24, eocd.centralDirRecords, true);
view.setBigUint64(32, eocd.totalRecords, true);
view.setBigUint64(40, eocd.centralDirSize, true);
view.setBigUint64(48, eocd.centralDirOffset, true);
return new Uint8Array(buffer);
}
function serializeZip64EndOfCentralDirectoryLocator(locator) {
const buffer = new ArrayBuffer(20);
const view = new DataView(buffer);
view.setUint32(0, locator.signature, true);
view.setUint32(4, locator.zip64EndDisk, true);
view.setBigUint64(8, locator.zip64EndOffset, true);
view.setUint32(16, locator.totalDisks, true);
return new Uint8Array(buffer);
}
function serializeDataDescriptor(descriptor) {
const hasSignature = descriptor.signature !== void 0;
const size = hasSignature ? 16 : 12;
const buffer = new ArrayBuffer(size);
const view = new DataView(buffer);
let offset = 0;
if (hasSignature) {
view.setUint32(offset, descriptor.signature, true);
offset += 4;
}
view.setUint32(offset, descriptor.crc32, true);
view.setUint32(offset + 4, descriptor.compressedSize, true);
view.setUint32(offset + 8, descriptor.uncompressedSize, true);
return new Uint8Array(buffer);
}
function serializeZip64DataDescriptor(descriptor) {
const hasSignature = descriptor.signature !== void 0;
const size = hasSignature ? 24 : 20;
const buffer = new ArrayBuffer(size);
const view = new DataView(buffer);
let offset = 0;
if (hasSignature) {
view.setUint32(offset, descriptor.signature, true);
offset += 4;
}
view.setUint32(offset, descriptor.crc32, true);
view.setBigUint64(offset + 4, descriptor.compressedSize, true);
view.setBigUint64(offset + 12, descriptor.uncompressedSize, true);
return new Uint8Array(buffer);
}
// src/crc32.ts
var crcTable = null;
function generateCrcTable() {
if (crcTable)
return crcTable;
crcTable = new Uint32Array(256);
for (let i = 0; i < 256; i++) {
let crc = i;
for (let j = 0; j < 8; j++) {
if (crc & 1) {
crc = crc >>> 1 ^ 3988292384;
} else {
crc = crc >>> 1;
}
}
crcTable[i] = crc;
}
return crcTable;
}
function crc32(data, crc = 0) {
const table = generateCrcTable();
crc = crc ^ 4294967295;
for (let i = 0; i < data.length; i++) {
crc = table[(crc ^ data[i]) & 255] ^ crc >>> 8;
}
return (crc ^ 4294967295) >>> 0;
}
var CRC32Stream = class {
constructor() {
this.crc = 0;
this.table = generateCrcTable();
this.crc = 4294967295;
}
/**
* Update CRC32 with new data chunk
*/
update(data) {
for (let i = 0; i < data.length; i++) {
this.crc = this.table[(this.crc ^ data[i]) & 255] ^ this.crc >>> 8;
}
}
/**
* Get the final CRC32 value
*/
digest() {
return (this.crc ^ 4294967295) >>> 0;
}
/**
* Reset the CRC32 calculator
*/
reset() {
this.crc = 4294967295;
}
/**
* Get current CRC32 value without finalization
*/
getCurrentValue() {
return (this.crc ^ 4294967295) >>> 0;
}
};
// src/logger.ts
var import_isomorphic_logger = require("@lyleunderwood/isomorphic-logger");
var logger = (0, import_isomorphic_logger.createLogger)({
level: process.env.LOG_LEVEL || "info"
});
var logger_default = logger;
// src/compression.ts
async function compressStore(data) {
const crc322 = new CRC32Stream();
crc322.update(data);
return {
compressedData: data,
crc32: crc322.digest(),
compressedSize: data.length,
uncompressedSize: data.length
};
}
async function compressDeflate(data) {
const crc32Stream = new CRC32Stream();
crc32Stream.update(data);
try {
const zlib = await import("zlib").catch(() => null);
if (zlib) {
const compressed = zlib.deflateRawSync(data);
return {
compressedData: new Uint8Array(compressed),
crc32: crc32Stream.digest(),
compressedSize: compressed.length,
uncompressedSize: data.length
};
}
if (typeof CompressionStream !== "undefined") {
const stream = new CompressionStream("deflate-raw");
const writer = stream.writable.getWriter();
const reader = stream.readable.getReader();
writer.write(data);
writer.close();
const chunks = [];
let result;
while (!(result = await reader.read()).done) {
chunks.push(result.value);
}
const compressed = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
let offset = 0;
for (const chunk of chunks) {
compressed.set(chunk, offset);
offset += chunk.length;
}
return {
compressedData: compressed,
crc32: crc32Stream.digest(),
compressedSize: compressed.length,
uncompressedSize: data.length
};
}
} catch (error) {
logger.warn({ error: error.message, stack: error.stack }, "Deflate compression failed, falling back to store method");
}
return compressStore(data);
}
function createCompressionStream(inputStream, method = "deflate") {
const crc32Stream = new CRC32Stream();
let compressedSize = 0;
let uncompressedSize = 0;
let crc32Resolver;
let compressedSizeResolver;
let uncompressedSizeResolver;
const crc32Promise = new Promise((resolve) => {
crc32Resolver = resolve;
});
const compressedSizePromise = new Promise((resolve) => {
compressedSizeResolver = resolve;
});
const uncompressedSizePromise = new Promise((resolve) => {
uncompressedSizeResolver = resolve;
});
const webStream = "getReader" in inputStream ? inputStream : new ReadableStream({
start(controller) {
const nodeStream = inputStream;
nodeStream.on("data", (chunk) => {
const uint8Array = new Uint8Array(chunk);
controller.enqueue(uint8Array);
});
nodeStream.on("end", () => controller.close());
nodeStream.on("error", (err) => controller.error(err));
}
});
if (method === "store") {
const storeTransform = new TransformStream({
transform(chunk, controller) {
crc32Stream.update(chunk);
uncompressedSize += chunk.length;
compressedSize += chunk.length;
controller.enqueue(chunk);
},
flush() {
crc32Resolver(crc32Stream.digest());
compressedSizeResolver(compressedSize);
uncompressedSizeResolver(uncompressedSize);
}
});
const readable = webStream.pipeThrough(storeTransform);
return {
readable,
crc32Promise,
compressedSizePromise,
uncompressedSizePromise
};
} else {
return createDeflateCompressionStream(
webStream,
crc32Stream,
crc32Resolver,
compressedSizeResolver,
uncompressedSizeResolver,
crc32Promise,
compressedSizePromise,
uncompressedSizePromise
);
}
}
function createDeflateCompressionStream(webStream, crc32Stream, crc32Resolver, compressedSizeResolver, uncompressedSizeResolver, crc32Promise, compressedSizePromise, uncompressedSizePromise) {
let compressedSize = 0;
let uncompressedSize = 0;
if (typeof process !== "undefined" && process.versions && process.versions.node) {
try {
const crcTransform = new TransformStream({
transform(chunk, controller) {
crc32Stream.update(chunk);
uncompressedSize += chunk.length;
controller.enqueue(chunk);
}
});
const sizeCounterTransform = new TransformStream({
transform(chunk, controller) {
compressedSize += chunk.length;
controller.enqueue(chunk);
},
flush() {
crc32Resolver(crc32Stream.digest());
compressedSizeResolver(compressedSize);
uncompressedSizeResolver(uncompressedSize);
}
});
const deflateStream = createNodeDeflateStream();
const readable = webStream.pipeThrough(crcTransform).pipeThrough(deflateStream).pipeThrough(sizeCounterTransform);
return {
readable,
crc32Promise,
compressedSizePromise,
uncompressedSizePromise
};
} catch (error) {
logger.warn({ error: error.message, stack: error.stack }, "Node.js deflate failed, falling back to browser compression");
}
}
if (typeof CompressionStream !== "undefined") {
try {
const crcTransform = new TransformStream({
transform(chunk, controller) {
crc32Stream.update(chunk);
uncompressedSize += chunk.length;
controller.enqueue(chunk);
}
});
const sizeCounterTransform = new TransformStream({
transform(chunk, controller) {
compressedSize += chunk.length;
controller.enqueue(chunk);
},
flush() {
crc32Resolver(crc32Stream.digest());
compressedSizeResolver(compressedSize);
uncompressedSizeResolver(uncompressedSize);
}
});
const deflateStream = new CompressionStream("deflate-raw");
const readable = webStream.pipeThrough(crcTransform).pipeThrough(deflateStream).pipeThrough(sizeCounterTransform);
return {
readable,
crc32Promise,
compressedSizePromise,
uncompressedSizePromise
};
} catch (error) {
logger.warn({ error: error.message, stack: error.stack }, "Browser compression failed, falling back to store method");
}
}
logger.warn("No compression available, using store method");
return createCompressionStream(webStream, "store");
}
function createNodeDeflateStream() {
let deflate = null;
let streamClosed = false;
return new TransformStream({
async start(controller) {
try {
const zlib = await import("zlib");
deflate = zlib.createDeflateRaw();
deflate.on("data", (chunk) => {
if (!streamClosed) {
controller.enqueue(new Uint8Array(chunk));
}
});
deflate.on("end", () => {
if (!streamClosed) {
streamClosed = true;
controller.terminate();
}
});
deflate.on("error", (err) => {
if (!streamClosed) {
streamClosed = true;
controller.error(err);
}
});
} catch (error) {
controller.error(error);
}
},
transform(chunk, controller) {
if (deflate && !streamClosed) {
deflate.write(Buffer.from(chunk));
} else if (!deflate) {
controller.error(new Error("Deflate stream not initialized"));
}
},
flush(controller) {
return new Promise((resolve, reject) => {
if (!deflate || streamClosed) {
resolve();
return;
}
const onEnd = () => {
deflate.removeListener("error", onError);
if (!streamClosed) {
streamClosed = true;
controller.terminate();
}
resolve();
};
const onError = (err) => {
deflate.removeListener("end", onEnd);
if (!streamClosed) {
streamClosed = true;
controller.error(err);
}
reject(err);
};
deflate.once("end", onEnd);
deflate.once("error", onError);
deflate.end();
});
}
});
}
// src/entry-buffer.ts
var EntryBuffer = class {
constructor(entry, index, compressionMethod = "store", maxBufferSize = 1024 * 1024) {
this.entry = entry;
this.index = index;
this.compressionMethod = compressionMethod;
this.chunks = [];
this.totalSize = 0;
this.crc32Stream = new CRC32Stream();
this.state = "pending" /* PENDING */;
this.readPromise = null;
this.readResolver = null;
this.readError = null;
this.metadata = {};
// Backpressure coordination
this.waitingReaders = [];
this.maxBufferSize = maxBufferSize;
}
/**
* Get current entry state
*/
getState() {
return this.state;
}
/**
* Check if entry is ready to be written
*/
isReady() {
return this.state === "ready" /* READY */;
}
/**
* Check if entry has been completely read
*/
isReadComplete() {
return this.state === "ready" /* READY */ || this.state === "writing" /* WRITING */ || this.state === "completed" /* COMPLETED */ || this.state === "error" /* ERROR */;
}
/**
* Get buffered size in bytes
*/
getBufferedSize() {
const actualSize = this.chunks.reduce((total, chunk) => total + chunk.data.length, 0);
if (actualSize !== this.totalSize) {
logger_default.debug({ actualSize, totalSize: this.totalSize }, "Buffer size mismatch detected, correcting");
this.totalSize = actualSize;
}
return this.totalSize;
}
/**
* Check if buffer has space for more data
*/
hasBufferSpace() {
return this.totalSize < this.maxBufferSize;
}
/**
* Start reading from the entry's data stream
*/
async startReading() {
if (this.state !== "pending" /* PENDING */) {
throw new Error(`Cannot start reading entry in state: ${this.state}`);
}
this.state = "reading" /* READING */;
this.readPromise = this.performRead();
return this.readPromise;
}
/**
* Wait for entry to be ready for writing
*/
async waitForReady() {
if (this.state === "ready" /* READY */) {
return;
}
if (this.state === "error" /* ERROR */) {
throw this.readError || new Error("Entry reading failed");
}
if (this.readPromise) {
await this.readPromise;
}
const currentState = this.state;
switch (currentState) {
case "error" /* ERROR */:
throw this.readError || new Error("Entry reading failed");
case "ready" /* READY */:
return;
default:
throw new Error(`Unexpected state after reading: ${currentState}`);
}
}
/**
* Read the next chunk from the buffer
*/
readChunk() {
if (this.chunks.length === 0) {
return null;
}
const chunk = this.chunks.shift();
if (chunk) {
this.totalSize -= chunk.data.length;
}
return chunk || null;
}
/**
* Get entry metadata
*/
getMetadata() {
if (!this.isReadComplete()) {
throw new Error("Metadata not available until reading is complete");
}
return {
crc32: this.metadata.crc32,
compressedSize: this.metadata.compressedSize,
uncompressedSize: this.metadata.uncompressedSize,
localHeaderOffset: this.metadata.localHeaderOffset || 0,
compressionMethod: this.compressionMethod
};
}
/**
* Set the local header offset
*/
setLocalHeaderOffset(offset) {
this.metadata.localHeaderOffset = offset;
}
/**
* Mark entry as being written
*/
startWriting() {
if (this.state !== "ready" /* READY */) {
throw new Error(`Cannot start writing entry in state: ${this.state}`);
}
this.state = "writing" /* WRITING */;
}
/**
* Mark entry as completed
*/
markCompleted() {
if (this.state !== "writing" /* WRITING */) {
throw new Error(`Cannot mark entry completed from state: ${this.state}`);
}
this.state = "completed" /* COMPLETED */;
}
/**
* Get total uncompressed size
*/
getUncompressedSize() {
return this.metadata.uncompressedSize || 0;
}
/**
* Get total compressed size (same as uncompressed for store method)
*/
getCompressedSize() {
return this.metadata.compressedSize || 0;
}
/**
* Get CRC32 checksum
*/
getCRC32() {
return this.metadata.crc32 || 0;
}
/**
* Perform the actual reading from the stream
*/
async performRead() {
logger_default.debug({ entryName: this.entry.name }, "Starting to read entry data");
try {
logger_default.debug({ entryName: this.entry.name }, "Reading entry data");
const inputStream = "getReader" in this.entry.data ? this.entry.data : this.createWebStreamFromNodeStream(this.entry.data);
logger_default.debug("Starting to read entry chunks");
const compressionResult = createCompressionStream(inputStream, this.compressionMethod);
const reader = compressionResult.readable.getReader();
logger_default.debug("Entry chunk reader created");
let chunkOffset = 0;
try {
while (true) {
logger_default.debug("Reading entry chunk");
const { done, value } = await reader.read();
logger_default.debug({ done }, "Entry chunk read");
if (done)
break;
if (!this.hasBufferSpace()) {
logger_default.debug("Waiting for buffer space");
await this.waitForBufferSpace();
logger_default.debug("Buffer space available");
}
this.chunks.push({
data: value,
offset: chunkOffset
});
this.totalSize += value.length;
chunkOffset += value.length;
}
} finally {
logger_default.debug("Releasing entry chunk reader");
reader.releaseLock();
}
const [crc322, compressedSize, uncompressedSize] = await Promise.all([
compressionResult.crc32Promise,
compressionResult.compressedSizePromise,
compressionResult.uncompressedSizePromise
]);
this.metadata.crc32 = crc322;
this.metadata.compressedSize = compressedSize;
this.metadata.uncompressedSize = uncompressedSize;
this.state = "ready" /* READY */;
logger_default.debug({ crc32: crc322, compressedSize, uncompressedSize }, "Entry read complete");
if (this.readResolver) {
logger_default.debug("Calling read resolver");
this.readResolver();
}
} catch (error) {
this.readError = error instanceof Error ? error : new Error(String(error));
this.state = "error" /* ERROR */;
if (this.readResolver) {
this.readResolver();
}
}
}
/**
* Wait for buffer space to become available
*/
async waitForBufferSpace() {
if (this.hasBufferSpace()) {
return;
}
return new Promise((resolve) => {
if (this.hasBufferSpace()) {
resolve();
return;
}
this.waitingReaders.push(resolve);
});
}
/**
* Notify that buffer space has been freed up (called when chunks are consumed)
*/
notifySpaceFreed() {
if (this.waitingReaders.length > 0) {
const hasSpace = this.hasBufferSpace();
if (hasSpace) {
const resolvers = this.waitingReaders.splice(0);
logger_default.debug({
bufferedSize: this.getBufferedSize(),
maxSize: this.maxBufferSize,
waitingReaders: resolvers.length
}, "Notifying waiting readers of available buffer space");
resolvers.forEach((resolve) => {
try {
resolve();
} catch (error) {
logger_default.error({ error }, "Error resolving waiting reader");
}
});
}
}
}
/**
* Create a Web ReadableStream from a Node.js stream
*/
createWebStreamFromNodeStream(nodeStream) {
logger_default.debug("Converting Node.js stream to Web Stream");
return new ReadableStream({
start(controller) {
nodeStream.on("data", (chunk) => {
const uint8Array = new Uint8Array(chunk);
controller.enqueue(uint8Array);
});
nodeStream.on("end", () => {
controller.close();
});
nodeStream.on("error", (err) => {
controller.error(err);
});
},
cancel() {
if ("destroy" in nodeStream && typeof nodeStream.destroy === "function") {
nodeStream.destroy();
}
}
});
}
};
// src/parallel-reader.ts
var ParallelReader = class {
constructor(options = {}) {
this.entryBuffers = [];
this.readingPromises = /* @__PURE__ */ new Map();
this.activeReads = 0;
this.options = {
maxBufferSize: 1024 * 1024,
// 1MB per entry
maxConcurrentReads: 10,
// Max concurrent reads
compression: "store",
// Default compression
...options
};
}
/**
* Add an entry to be read in parallel
*/
addEntry(entry) {
const index = this.entryBuffers.length;
const entryBuffer = new EntryBuffer(
entry,
index,
this.options.compression,
this.options.maxBufferSize
);
this.entryBuffers.push(entryBuffer);
this.startReadingIfPossible(entryBuffer);
return entryBuffer;
}
/**
* Get all entry buffers
*/
getEntryBuffers() {
return [...this.entryBuffers];
}
/**
* Get entry buffer by index
*/
getEntryBuffer(index) {
return this.entryBuffers[index];
}
/**
* Get the next ready entry in order
*/
getNextReadyEntry() {
for (const buffer of this.entryBuffers) {
if (buffer.getState() === "ready" /* READY */) {
return buffer;
}
if (buffer.getState() !== "completed" /* COMPLETED */) {
return null;
}
}
return null;
}
/**
* Check if there are any entries still being read
*/
hasActiveReads() {
return this.activeReads > 0 || this.readingPromises.size > 0;
}
/**
* Wait for the next entry to become ready
*/
async waitForNextReady() {
const nextReady = this.getNextReadyEntry();
if (nextReady) {
return nextReady;
}
let nextEntry = null;
for (const buffer of this.entryBuffers) {
if (buffer.getState() === "completed" /* COMPLETED */) {
continue;
}
nextEntry = buffer;
break;
}
if (!nextEntry) {
return null;
}
if (nextEntry.getState() === "pending" /* PENDING */) {
this.startReadingIfPossible(nextEntry);
}
if (nextEntry.getState() !== "ready" /* READY */) {
await nextEntry.waitForReady();
}
return nextEntry.isReady() ? nextEntry : null;
}
/**
* Wait for all entries to complete reading
*/
async waitForAllComplete() {
for (const buffer of this.entryBuffers) {
if (buffer.getState() === "pending" /* PENDING */) {
logger_default.debug({ entryName: buffer.entry.name }, "Starting to read pending entry");
this.startReadingIfPossible(buffer);
}
}
const promises = Array.from(this.readingPromises.values());
if (promises.length > 0) {
logger_default.debug({ count: promises.length }, "Waiting for all reading promises to complete");
await Promise.all(promises);
}
}
/**
* Get statistics about reading progress
*/
getStats() {
const stats = {
total: this.entryBuffers.length,
pending: 0,
reading: 0,
ready: 0,
writing: 0,
completed: 0,
errors: 0
};
for (const buffer of this.entryBuffers) {
switch (buffer.getState()) {
case "pending" /* PENDING */:
stats.pending++;
break;
case "reading" /* READING */:
stats.reading++;
break;
case "ready" /* READY */:
stats.ready++;
break;
case "writing" /* WRITING */:
stats.writing++;
break;
case "completed" /* COMPLETED */:
stats.completed++;
break;
case "error" /* ERROR */:
stats.errors++;
break;
}
}
return stats;
}
/**
* Start reading an entry if we have capacity
*/
startReadingIfPossible(entryBuffer) {
if (entryBuffer.getState() !== "pending" /* PENDING */) {
return;
}
if (this.activeReads >= this.options.maxConcurrentReads) {
return;
}
this.activeReads++;
logger_default.debug({ entryName: entryBuffer.entry.name }, "Starting to read entry");
const readPromise = entryBuffer.startReading().finally(() => {
logger_default.debug({ entryName: entryBuffer.entry.name }, "Completed reading entry");
this.activeReads--;
this.readingPromises.delete(entryBuffer.index);
this.startNextPendingEntry();
});
this.readingPromises.set(entryBuffer.index, readPromise);
}
/**
* Try to start reading the next pending entry
*/
startNextPendingEntry() {
if (this.activeReads >= this.options.maxConcurrentReads) {
return;
}
for (const buffer of this.entryBuffers) {
if (buffer.getState() === "pending" /* PENDING */) {
this.startReadingIfPossible(buffer);
break;
}
}
}
};
// src/write-queue.ts
var WriteQueue = class {
constructor(options = {}) {
this.outputController = null;
this.currentOffset = 0;
this.centralDirectoryEntries = [];
this.writeInProgress = false;
this.options = {
compression: "store",
...options
};
logger.debug({ options: this.options }, "WriteQueue initialized");
}
/**
* Set the output controller for streaming data
*/
setOutputController(controller) {
this.outputController = controller;
}
/**
* Write an entry buffer to the output stream
*/
async writeEntry(entryBuffer) {
if (this.writeInProgress) {
throw new Error("Another write operation is already in progress");
}
if (!entryBuffer.isReady()) {
throw new Error("Entry buffer is not ready for writing");
}
const entryName = entryBuffer.entry.name;
logger.debug({ entryName, currentOffset: this.currentOffset }, "Starting to write buffered entry");
this.writeInProgress = true;
try {
entryBuffer.startWriting();
await this.performEntryWrite(entryBuffer);
entryBuffer.markCompleted();
logger.debug({ entryName, finalOffset: this.currentOffset }, "Completed writing buffered entry");
} catch (error) {
logger.error({ entryName, error: error.message, stack: error.stack }, "Error writing buffered entry");
throw error;
} finally {
this.writeInProgress = false;
}
}
/**
* Write a direct stream entry immediately to the output stream
*/
async writeDirectStreamEntry(directEntry) {
if (this.writeInProgress) {
throw new Error("Another write operation is already in progress");
}
if (!directEntry.isReadyForStreaming()) {
throw new Error("Direct stream entry is not ready for streaming");
}
const entryName = directEntry.entry.name;
logger.debug({ entryName, currentOffset: this.currentOffset }, "Starting to write direct stream entry");
this.writeInProgress = true;
try {
await this.performDirectStreamWrite(directEntry);
directEntry.markCompleted();
logger.debug({ entryName, finalOffset: this.currentOffset }, "Completed writing direct stream entry");
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
logger.error({ entryName, error: err.message, stack: err.stack }, "Error writing direct stream entry");
directEntry.markError(err);
throw err;
} finally {
this.writeInProgress = false;
}
}
/**
* Write the central directory to the output stream
*/
async writeCentralDirectory() {
const centralDirStart = this.currentOffset;
const entryCount = this.centralDirectoryEntries.length;
logger.debug({ entryCount, centralDirStart }, "Writing central directory");
for (let i = 0; i < this.centralDirectoryEntries.length; i++) {
const entry = this.centralDirectoryEntries[i];
logger.debug({ entryIndex: i, entryName: entry?.filename }, "Writing central directory entry");
if (!entry) {
throw new Error("Central directory entry is missing");
}
const serialized = serializeCentralDirectoryHeader(entry);
await this.writeToOutput(serialized);
}
const centralDirSize = this.currentOffset - centralDirStart;
logger.debug({ centralDirStart, centralDirSize }, "Central directory written successfully");
return {
centralDirOffset: centralDirStart,
centralDirSize
};
}
/**
* Get the current offset in the output stream
*/
getCurrentOffset() {
return this.currentOffset;
}
/**
* Get the number of entries written
*/
getEntryCount() {
return this.centralDirectoryEntries.length;
}
/**
* Check if any write operations are in progress
*/
isWriteInProgress() {
return this.writeInProgress;
}
/**
* Perform immediate direct stream write operation
*/
async performDirectStreamWrite(directEntry) {
directEntry.setLocalHeaderOffset(this.currentOffset);
const localHeader = this.createDirectStreamLocalFileHeader(directEntry);
const headerSize = serializeLocalFileHeader(localHeader).length;
logger.debug({ entryName: directEntry.entry.name, headerSize }, "Writing direct stream local file header");
await this.writeToOutput(serializeLocalFileHeader(localHeader));
const dataStream = directEntry.startDirectStreaming();
const reader = dataStream.getReader();
let streamedDataSize = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
streamedDataSize += value.length;
await this.writeToOutput(value);
}
} finally {
reader.releaseLock();
}
logger.debug({ entryName: directEntry.entry.name, streamedDataSize }, "Direct stream data written");
const dataDescriptor = this.createDirectStreamDataDescriptor(directEntry);
logger.debug({ entryName: directEntry.entry.name, descriptorSize: dataDescriptor.length }, "Writing direct stream data descriptor");
await this.writeToOutput(dataDescriptor);
const centralDirEntry = this.createDirectStreamCentralDirectoryHeader(directEntry);
this.centralDirectoryEntries.push(centralDirEntry);
}
/**
* Perform the actual entry write operation
*/
async performEntryWrite(entryBuffer) {
entryBuffer.setLocalHeaderOffset(this.currentOffset);
const localHeader = this.createLocalFileHeader(entryBuffer);
logger.debug({ entryName: entryBuffer.entry.name, headerSize: serializeLocalFileHeader(localHeader).length }, "Writing local file header");
await this.writeToOutput(serializeLocalFileHeader(localHeader));
let chunk;
let dataSize = 0;
while ((chunk = entryBuffer.readChunk()) !== null) {
dataSize += chunk.data.length;
await this.writeToOutput(chunk.data);
entryBuffer.notifySpaceFreed();
}
logger.debug({ entryName: entryBuffer.entry.name, dataSize }, "Entry data written");
if (localHeader.flags & ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR) {
const dataDescriptor = this.createDataDescriptor(entryBuffer);
logger.debug({ entryName: entryBuffer.entry.name, descriptorSize: dataDescriptor.length }, "Writing data descriptor");
await this.writeToOutput(dataDescriptor);
}
const centralDirEntry = this.createCentralDirectoryHeader(entryBuffer);
this.centralDirectoryEntries.push(centralDirEntry);
}
/**
* Create local file header for entry
*/
createLocalFileHeader(entryBuffer) {
const entry = entryBuffer.entry;
const filename = new TextEncoder().encode(entry.name);
const lastModified = entry.lastModified || /* @__PURE__ */ new Date();
const { date, time } = dateToDosDateTime(lastModified);
const flags = ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8;
const extraField = new Uint8Array(0);
return {
signature: ZIP_CONSTANTS.LOCAL_FILE_HEADER_SIGNATURE,
versionNeeded: ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT,
flags,
compressionMethod: entryBuffer.getMetadata().compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE,
lastModTime: time,
lastModDate: date,
crc32: 0,
// Will be in data descriptor
compressedSize: 0,
// Will be in data descriptor
uncompressedSize: 0,
// Will be in data descriptor
filenameLength: filename.length,
extraFieldLength: extraField.length,
filename,
extraField
};
}
/**
* Create data descriptor for entry
*/
createDataDescriptor(entryBuffer) {
const metadata = entryBuffer.getMetadata();
const needsZip64Format = needsZip64(
metadata.uncompressedSize,
metadata.compressedSize,
0,
0,
0
);
if (needsZip64Format) {
const descriptor = {
signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE,
crc32: metadata.crc32,
compressedSize: BigInt(metadata.compressedSize),
uncompressedSize: BigInt(metadata.uncompressedSize)
};
return serializeZip64DataDescriptor(descriptor);
} else {
const descriptor = {
signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE,
crc32: metadata.crc32,
compressedSize: metadata.compressedSize,
uncompressedSize: metadata.uncompressedSize
};
return serializeDataDescriptor(descriptor);
}
}
/**
* Create central directory header for entry
*/
createCentralDirectoryHeader(entryBuffer) {
const entry = entryBuffer.entry;
const metadata = entryBuffer.getMetadata();
const filename = new TextEncoder().encode(entry.name);
const comment = new TextEncoder().encode(entry.comment || "");
const lastModified = entry.lastModified || /* @__PURE__ */ new Date();
const { date, time } = dateToDosDateTime(lastModified);
const needsZip64Format = needsZip64(
metadata.uncompressedSize,
metadata.compressedSize,
metadata.localHeaderOffset,
0,
0
);
let extraField = new Uint8Array(0);
let compressedSize = metadata.compressedSize;
let uncompressedSize = metadata.uncompressedSize;
let localHeaderOffset = metadata.localHeaderOffset;
if (needsZip64Format) {
extraField = createZip64ExtraField(
BigInt(metadata.uncompressedSize),
BigInt(metadata.compressedSize),
BigInt(metadata.localHeaderOffset)
);
compressedSize = ZIP_CONSTANTS.ZIP64_LIMIT;
uncompressedSize = ZIP_CONSTANTS.ZIP64_LIMIT;
localHeaderOffset = ZIP_CONSTANTS.ZIP64_LIMIT;
}
return {
signature: ZIP_CONSTANTS.CENTRAL_DIRECTORY_SIGNATURE,
versionMadeBy: ZIP_CONSTANTS.VERSION_MADE_BY,
versionNeeded: needsZip64Format ? ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT_ZIP64 : ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT,
flags: ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8,
compressionMethod: metadata.compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE,
lastModTime: time,
lastModDate: date,
crc32: metadata.crc32,
compressedSize,
uncompressedSize,
filenameLength: filename.length,
extraFieldLength: extraField.length,
commentLength: comment.length,
diskNumber: 0,
internalAttributes: 0,
externalAttributes: createExternalAttributes(entry.permissions, entry.name.endsWith("/")),
localHeaderOffset,
filename,
extraField,
comment
};
}
/**
* Create local file header for direct stream entry
*/
createDirectStreamLocalFileHeader(directEntry) {
const entry = directEntry.entry;
const metadata = directEntry.getMetadata();
const filename = new TextEncoder().encode(entry.name);
const lastModified = entry.lastModified || /* @__PURE__ */ new Date();
const { date, time } = dateToDosDateTime(lastModified);
const flags = ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8;
const extraField = new Uint8Array(0);
return {
signature: ZIP_CONSTANTS.LOCAL_FILE_HEADER_SIGNATURE,
versionNeeded: ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT,
flags,
compressionMethod: metadata.compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE,
lastModTime: time,
lastModDate: date,
crc32: 0,
// Will be in data descriptor
compressedSize: 0,
// Will be in data descriptor
uncompressedSize: 0,
// Will be in data descriptor
filenameLength: filename.length,
extraFieldLength: extraField.length,
filename,
extraField
};
}
/**
* Create data descriptor for direct stream entry
*/
createDirectStreamDataDescriptor(directEntry) {
const metadata = directEntry.getMetadata();
const needsZip64Format = needsZip64(
metadata.uncompressedSize,
metadata.compressedSize,
0,
0,
0
);
if (needsZip64Format) {
const descriptor = {
signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE,
crc32: metadata.crc32,
compressedSize: BigInt(metadata.compressedSize),
uncompressedSize: BigInt(metadata.uncompressedSize)
};
return serializeZip64DataDescriptor(descriptor);
} else {
const descriptor = {
signature: ZIP_CONSTANTS.DATA_DESCRIPTOR_SIGNATURE,
crc32: metadata.crc32,
compressedSize: metadata.compressedSize,
uncompressedSize: metadata.uncompressedSize
};
return serializeDataDescriptor(descriptor);
}
}
/**
* Create central directory header for direct stream entry
*/
createDirectStreamCentralDirectoryHeader(directEntry) {
const entry = directEntry.entry;
const metadata = directEntry.getMetadata();
const filename = new TextEncoder().encode(entry.name);
const comment = new TextEncoder().encode(entry.comment || "");
const lastModified = entry.lastModified || /* @__PURE__ */ new Date();
const { date, time } = dateToDosDateTime(lastModified);
const needsZip64Format = needsZip64(
metadata.uncompressedSize,
metadata.compressedSize,
metadata.localHeaderOffset,
0,
0
);
let extraField = new Uint8Array(0);
let compressedSize = metadata.compressedSize;
let uncompressedSize = metadata.uncompressedSize;
let localHeaderOffset = metadata.localHeaderOffset;
if (needsZip64Format) {
extraField = createZip64ExtraField(
BigInt(metadata.uncompressedSize),
BigInt(metadata.compressedSize),
BigInt(metadata.localHeaderOffset)
);
compressedSize = ZIP_CONSTANTS.ZIP64_LIMIT;
uncompressedSize = ZIP_CONSTANTS.ZIP64_LIMIT;
localHeaderOffset = ZIP_CONSTANTS.ZIP64_LIMIT;
}
return {
signature: ZIP_CONSTANTS.CENTRAL_DIRECTORY_SIGNATURE,
versionMadeBy: ZIP_CONSTANTS.VERSION_MADE_BY,
versionNeeded: needsZip64Format ? ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT_ZIP64 : ZIP_CONSTANTS.VERSION_NEEDED_EXTRACT,
flags: ZIP_CONSTANTS.FLAG_DATA_DESCRIPTOR | ZIP_CONSTANTS.FLAG_UTF8,
compressionMethod: metadata.compressionMethod === "store" ? ZIP_CONSTANTS.COMPRESSION_STORE : ZIP_CONSTANTS.COMPRESSION_DEFLATE,
lastModTime: time,
lastModDate: date,
crc32: metadata.crc32,
compressedSize,
uncompressedSize,
filenameLength: filename.length,
extraFieldLength: extraField.length,
commentLength: comment.length,
diskNumber: 0,
internalAttributes: 0,
externalAttributes: createExternalAttributes(entry.permissions, entry.name.endsWith("/")),
localHeaderOffset,
filename,
extraField,
comment
};
}
/**
* Write data to the output stream
*/
async writeToOutput(data) {
if (this.outputController) {
this.outputController.enqueue(data);
this.currentOffset += data.length;
}
}
};
// src/direct-stream-entry.ts
var DirectStreamEntry = class {
constructor(entry, index, compressionMethod) {
this.entry = entry;
this.index = index;
this.state = "pending" /* PENDING */;
this.streamReader = null;
this.error = null;
if (entry.preCompressed) {
this.metadata = {
crc32: entry.crc32,
compressedSize: entry.compressedSize,
uncompressedSize: entry.uncompressedSize,
localHeaderOffset: 0,
compressionMethod: "deflate"
};
} else {
const size = entry.size || 0;
this.metadata = {
crc32: entry.crc32,
compressedSize: size,
uncompressedSize: size,
localHeaderOffset: 0,
compressionMethod: "store"
};
}
}
/**
* Get current state
*/
getState() {
return this.state;
}
/**
* Check if entry is ready for immediate streaming
*/
isReadyForStreaming() {
return this.state === "pending" /* PENDING */;
}
/**
* Check if streaming is complete
*/
isCompleted() {
return this.state === "completed" /* COMPLETED */;
}
/**
* Check if there was an error
*/
hasError() {
return this.state === "error" /* ERROR */;
}
/**
* Get error if any
*/
g