UNPKG

ipull

Version:

The only file downloader you'll ever need. For node.js and the browser, CLI and library for fast and reliable file downloads.

322 lines 13.9 kB
import ProgressStatusFile, { DownloadFlags, DownloadStatus } from "./progress-status-file.js"; import { ChunkStatus } from "../types.js"; import { EventEmitter } from "eventemitter3"; import { withLock } from "lifecycle-utils"; import switchProgram from "./download-programs/switch-program.js"; import { pushComment } from "./utils/push-comment.js"; import { uid } from "uid"; import { DownloaderProgramManager } from "./downloaderProgramManager.js"; const DEFAULT_CHUNKS_SIZE_FOR_CHUNKS_PROGRAM = 1024 * 1024 * 5; // 5MB const DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM = 1024 * 1024; // 1MB const DEFAULT_OPTIONS = { chunkSize: 0, parallelStreams: 3, autoIncreaseParallelStreams: true }; export default class DownloadEngineFile extends EventEmitter { file; options; _progress = { downloadId: "", part: 0, chunks: [], chunkSize: 0, parallelStreams: 0 }; _closed = false; _progressStatus; _activeStreamContext = {}; _activeProgram; _downloadStatus = DownloadStatus.NotStarted; _latestProgressDate = 0; constructor(file, options) { super(); this.file = file; this.options = { ...DEFAULT_OPTIONS, ...options }; this._progressStatus = new ProgressStatusFile(file.parts.length, file.localFileName, options.fetchStream.transferAction, this._createProgressFlags()); this._setDefaultByOptions(); this._initProgress(); } _setDefaultByOptions() { if (this.options.chunkSize === 0) { switch (this._programType) { case "chunks": this.options.chunkSize = DEFAULT_CHUNKS_SIZE_FOR_CHUNKS_PROGRAM; break; case "stream": default: this.options.chunkSize = DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM; break; } } } _createProgressFlags() { const flags = []; if (this.options.skipExisting) { flags.push(DownloadFlags.Existing); } return flags; } get downloadSize() { return this.file.parts.reduce((acc, part) => acc + part.size, 0); } get fileName() { return this.file.localFileName; } get status() { const thisStatus = this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this.downloadSize, this._downloadStatus, this.options.comment); const streamContexts = Object.values(this._activeStreamContext); thisStatus.retrying = streamContexts.some(c => c.isRetrying); thisStatus.retryingTotalAttempts = Math.max(0, ...streamContexts.map(x => x.retryingAttempts)); thisStatus.streamsNotResponding = streamContexts.reduce((acc, cur) => acc + (cur.isStreamNotResponding ? 1 : 0), 0); return thisStatus; } get _activePart() { return this.file.parts[this._progress.part]; } get _downloadedPartsSize() { return this.file.parts.slice(0, this._progress.part) .reduce((acc, part) => acc + part.size, 0); } get _activeDownloadedChunkSize() { return this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE).length * this._progress.chunkSize; } get transferredBytes() { if (this._downloadStatus === DownloadStatus.Finished) { return this.downloadSize; } const streamingBytes = Object.values(this._activeStreamContext) .reduce((acc, cur) => acc + cur.streamBytes, 0); const streamBytes = this._activeDownloadedChunkSize + streamingBytes; const streamBytesMin = Math.min(streamBytes, this._activePart.size || streamBytes); const allBytes = streamBytesMin + this._downloadedPartsSize; return Math.min(allBytes, this.downloadSize || allBytes); } get _programType() { if (this.options.programType && this.options.fetchStream.availablePrograms.includes(this.options.programType)) { return this.options.programType; } return this.options.fetchStream.defaultProgramType; } _chunksForPart(part, fill = ChunkStatus.NOT_STARTED) { const partInfo = this.file.parts[part]; if (partInfo.size === 0) { return [ChunkStatus.NOT_STARTED]; } const chunksCount = Math.ceil(partInfo.size / this.options.chunkSize); return new Array(chunksCount).fill(fill); } _initEventReloadStatus() { if (this._progress.part === this.file.parts.length - 1 && this._progress.chunks.every(c => c === ChunkStatus.COMPLETE)) { this._downloadStatus = DownloadStatus.Finished; } } _initProgress() { if (this.options.skipExisting) { this._progress.part = this.file.parts.length - 1; this._progress.chunks = this._chunksForPart(this._progress.part, ChunkStatus.COMPLETE); this._progress.chunkSize = this.options.chunkSize; this.options.comment = pushComment("Skipping existing", this.options.comment); } else if (this.file.downloadProgress) { this._progress = this.file.downloadProgress; this._progress.parallelStreams = this.options.parallelStreams; this._initEventReloadStatus(); } else { this._progress = { part: 0, downloadId: uid(), chunks: this._chunksForPart(0), chunkSize: this.options.chunkSize, parallelStreams: this.options.parallelStreams }; this._progressStatus.downloadId = this._progress.downloadId; } } async download() { if (this._downloadStatus === DownloadStatus.NotStarted) { this._downloadStatus = DownloadStatus.Active; } this._progressStatus.started(); this.emit("start"); await this.options.onStartedAsync?.(); this._sendProgressDownloadPart(); for (let i = this._progress.part; i < this.file.parts.length && !this.options.skipExisting; i++) { if (this._closed) return; // If we are starting a new part, we need to reset the progress if (i > this._progress.part || !this._activePart.acceptRange) { this._progress.part = i; this._progress.chunkSize = this.options.chunkSize; this._progress.parallelStreams = this.options.parallelStreams; this._progress.chunks = this._chunksForPart(i); } // Reset in progress chunks this._progress.chunks = this._progress.chunks.map(chunk => (chunk === ChunkStatus.COMPLETE ? ChunkStatus.COMPLETE : ChunkStatus.NOT_STARTED)); // Reset active stream progress this._activeStreamContext = {}; if (this._activePart.acceptRange) { this._activeProgram = switchProgram(this._progress, this._downloadSlice.bind(this), this._programType); let manager = null; if (this.options.autoIncreaseParallelStreams && this.options.fetchStream.supportDynamicStreamLength) { manager = new DownloaderProgramManager(this._activeProgram, this); } try { await this._activeProgram.download(); } finally { manager?.close(); } } else { const chunksToRead = this._activePart.size > 0 ? this._progress.chunks.length : Infinity; await this._downloadSlice(0, chunksToRead); } } // All parts are downloaded, we can clear the progress this._activeStreamContext = {}; this._latestProgressDate = 0; if (this._closed) return; this._progressStatus.finished(); this._downloadStatus = DownloadStatus.Finished; this._sendProgressDownloadPart(); this.emit("finished"); await this.options.onFinishAsync?.(); } async _downloadSlice(startChunk, endChunk) { const getContext = () => this._activeStreamContext[startChunk] ??= { streamBytes: 0, retryingAttempts: 0 }; const fetchState = this.options.fetchStream.withSubState({ chunkSize: this._progress.chunkSize, startChunk, endChunk: endChunk, lastChunkEndsFile: endChunk === Infinity || endChunk === this._progress.chunks.length, totalSize: this._activePart.size, url: this._activePart.downloadURL, rangeSupport: this._activePart.acceptRange, onProgress: (length) => { getContext().streamBytes = length; this._sendProgressDownloadPart(); } }); fetchState.addListener("retryingOn", () => { const context = getContext(); context.isRetrying = true; context.retryingAttempts++; this._sendProgressDownloadPart(); }); fetchState.addListener("retryingOff", () => { getContext().isRetrying = false; }); fetchState.addListener("streamNotRespondingOn", () => { getContext().isStreamNotResponding = true; this._sendProgressDownloadPart(); }); fetchState.addListener("streamNotRespondingOff", () => { getContext().isStreamNotResponding = false; }); const downloadedPartsSize = this._downloadedPartsSize; this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS; const allWrites = new Set(); let lastChunkSize = 0, lastInProgressIndex = startChunk; await fetchState.fetchChunks((chunks, writePosition, index) => { if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) { return; } const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunks); if (writePromise) { allWrites.add(writePromise); writePromise.then(() => { allWrites.delete(writePromise); }); } // if content length is 0, we do not know how many chunks we should have if (this._activePart.size === 0) { this._progress.chunks.push(ChunkStatus.NOT_STARTED); } this._progress.chunks[index] = ChunkStatus.COMPLETE; lastChunkSize = chunks.reduce((last, current) => last + current.length, 0); getContext().streamBytes = 0; void this._saveProgress(); const nextChunk = this._progress.chunks[index + 1]; const shouldReadNext = fetchState.state.endChunk - index > 1; // grater than 1, meaning there is a next chunk if (shouldReadNext) { if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) { return fetchState.close(); } this._progress.chunks[lastInProgressIndex = index + 1] = ChunkStatus.IN_PROGRESS; } }); if (this._progress.chunks[lastInProgressIndex] === ChunkStatus.IN_PROGRESS) { this._progress.chunks[lastInProgressIndex] = ChunkStatus.NOT_STARTED; } // On dynamic content length, we need to adjust the last chunk size if (this._activePart.size === 0) { this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize; this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE); } delete this._activeStreamContext[startChunk]; await Promise.all(allWrites); } _saveProgress() { const thisProgress = this._latestProgressDate = Date.now(); this._sendProgressDownloadPart(); if (!this._activePart.acceptRange) return; this.emit("save", this._progress); return withLock(this, "_saveLock", async () => { if (thisProgress === this._latestProgressDate && !this._closed && this._downloadStatus !== DownloadStatus.Finished) { await this.options.onSaveProgressAsync?.(this._progress); } }); } _sendProgressDownloadPart() { if (this._closed) return; this.emit("progress", this.status); } async pause() { if (this.options.fetchStream.paused) { return; } this._downloadStatus = DownloadStatus.Paused; this.options.fetchStream.emit("paused"); await this.options.onPausedAsync?.(); this._sendProgressDownloadPart(); } resume() { if (!this.options.fetchStream.paused) { return; } this._downloadStatus = DownloadStatus.Active; this.options.fetchStream.emit("resumed"); this.emit("resumed"); this._sendProgressDownloadPart(); } async close() { if (this._closed) return; if (this._downloadStatus !== DownloadStatus.Finished) { this._progressStatus.finished(); this._downloadStatus = DownloadStatus.Cancelled; this._sendProgressDownloadPart(); } this._closed = true; this._activeProgram?.abort(); await this.options.onCloseAsync?.(); await this.options.writeStream.close(); await this.options.fetchStream.close(); this.emit("closed"); } finished(comment) { if (comment) { this.options.comment = pushComment(comment, this.options.comment); } this._downloadStatus = DownloadStatus.Finished; } [Symbol.dispose]() { return this.close(); } } //# sourceMappingURL=download-engine-file.js.map