ipull
Version:
The only file downloader you'll ever need. For node.js and the browser, CLI and library for fast and reliable file downloads.
322 lines • 13.9 kB
JavaScript
import ProgressStatusFile, { DownloadFlags, DownloadStatus } from "./progress-status-file.js";
import { ChunkStatus } from "../types.js";
import { EventEmitter } from "eventemitter3";
import { withLock } from "lifecycle-utils";
import switchProgram from "./download-programs/switch-program.js";
import { pushComment } from "./utils/push-comment.js";
import { uid } from "uid";
import { DownloaderProgramManager } from "./downloaderProgramManager.js";
const DEFAULT_CHUNKS_SIZE_FOR_CHUNKS_PROGRAM = 1024 * 1024 * 5; // 5MB
const DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM = 1024 * 1024; // 1MB
const DEFAULT_OPTIONS = {
chunkSize: 0,
parallelStreams: 3,
autoIncreaseParallelStreams: true
};
export default class DownloadEngineFile extends EventEmitter {
file;
options;
_progress = {
downloadId: "",
part: 0,
chunks: [],
chunkSize: 0,
parallelStreams: 0
};
_closed = false;
_progressStatus;
_activeStreamContext = {};
_activeProgram;
_downloadStatus = DownloadStatus.NotStarted;
_latestProgressDate = 0;
constructor(file, options) {
super();
this.file = file;
this.options = { ...DEFAULT_OPTIONS, ...options };
this._progressStatus = new ProgressStatusFile(file.parts.length, file.localFileName, options.fetchStream.transferAction, this._createProgressFlags());
this._setDefaultByOptions();
this._initProgress();
}
_setDefaultByOptions() {
if (this.options.chunkSize === 0) {
switch (this._programType) {
case "chunks":
this.options.chunkSize = DEFAULT_CHUNKS_SIZE_FOR_CHUNKS_PROGRAM;
break;
case "stream":
default:
this.options.chunkSize = DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM;
break;
}
}
}
_createProgressFlags() {
const flags = [];
if (this.options.skipExisting) {
flags.push(DownloadFlags.Existing);
}
return flags;
}
get downloadSize() {
return this.file.parts.reduce((acc, part) => acc + part.size, 0);
}
get fileName() {
return this.file.localFileName;
}
get status() {
const thisStatus = this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this.downloadSize, this._downloadStatus, this.options.comment);
const streamContexts = Object.values(this._activeStreamContext);
thisStatus.retrying = streamContexts.some(c => c.isRetrying);
thisStatus.retryingTotalAttempts = Math.max(0, ...streamContexts.map(x => x.retryingAttempts));
thisStatus.streamsNotResponding = streamContexts.reduce((acc, cur) => acc + (cur.isStreamNotResponding ? 1 : 0), 0);
return thisStatus;
}
get _activePart() {
return this.file.parts[this._progress.part];
}
get _downloadedPartsSize() {
return this.file.parts.slice(0, this._progress.part)
.reduce((acc, part) => acc + part.size, 0);
}
get _activeDownloadedChunkSize() {
return this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE).length * this._progress.chunkSize;
}
get transferredBytes() {
if (this._downloadStatus === DownloadStatus.Finished) {
return this.downloadSize;
}
const streamingBytes = Object.values(this._activeStreamContext)
.reduce((acc, cur) => acc + cur.streamBytes, 0);
const streamBytes = this._activeDownloadedChunkSize + streamingBytes;
const streamBytesMin = Math.min(streamBytes, this._activePart.size || streamBytes);
const allBytes = streamBytesMin + this._downloadedPartsSize;
return Math.min(allBytes, this.downloadSize || allBytes);
}
get _programType() {
if (this.options.programType && this.options.fetchStream.availablePrograms.includes(this.options.programType)) {
return this.options.programType;
}
return this.options.fetchStream.defaultProgramType;
}
_chunksForPart(part, fill = ChunkStatus.NOT_STARTED) {
const partInfo = this.file.parts[part];
if (partInfo.size === 0) {
return [ChunkStatus.NOT_STARTED];
}
const chunksCount = Math.ceil(partInfo.size / this.options.chunkSize);
return new Array(chunksCount).fill(fill);
}
_initEventReloadStatus() {
if (this._progress.part === this.file.parts.length - 1 && this._progress.chunks.every(c => c === ChunkStatus.COMPLETE)) {
this._downloadStatus = DownloadStatus.Finished;
}
}
_initProgress() {
if (this.options.skipExisting) {
this._progress.part = this.file.parts.length - 1;
this._progress.chunks = this._chunksForPart(this._progress.part, ChunkStatus.COMPLETE);
this._progress.chunkSize = this.options.chunkSize;
this.options.comment = pushComment("Skipping existing", this.options.comment);
}
else if (this.file.downloadProgress) {
this._progress = this.file.downloadProgress;
this._progress.parallelStreams = this.options.parallelStreams;
this._initEventReloadStatus();
}
else {
this._progress = {
part: 0,
downloadId: uid(),
chunks: this._chunksForPart(0),
chunkSize: this.options.chunkSize,
parallelStreams: this.options.parallelStreams
};
this._progressStatus.downloadId = this._progress.downloadId;
}
}
async download() {
if (this._downloadStatus === DownloadStatus.NotStarted) {
this._downloadStatus = DownloadStatus.Active;
}
this._progressStatus.started();
this.emit("start");
await this.options.onStartedAsync?.();
this._sendProgressDownloadPart();
for (let i = this._progress.part; i < this.file.parts.length && !this.options.skipExisting; i++) {
if (this._closed)
return;
// If we are starting a new part, we need to reset the progress
if (i > this._progress.part || !this._activePart.acceptRange) {
this._progress.part = i;
this._progress.chunkSize = this.options.chunkSize;
this._progress.parallelStreams = this.options.parallelStreams;
this._progress.chunks = this._chunksForPart(i);
}
// Reset in progress chunks
this._progress.chunks = this._progress.chunks.map(chunk => (chunk === ChunkStatus.COMPLETE ? ChunkStatus.COMPLETE : ChunkStatus.NOT_STARTED));
// Reset active stream progress
this._activeStreamContext = {};
if (this._activePart.acceptRange) {
this._activeProgram = switchProgram(this._progress, this._downloadSlice.bind(this), this._programType);
let manager = null;
if (this.options.autoIncreaseParallelStreams && this.options.fetchStream.supportDynamicStreamLength) {
manager = new DownloaderProgramManager(this._activeProgram, this);
}
try {
await this._activeProgram.download();
}
finally {
manager?.close();
}
}
else {
const chunksToRead = this._activePart.size > 0 ? this._progress.chunks.length : Infinity;
await this._downloadSlice(0, chunksToRead);
}
}
// All parts are downloaded, we can clear the progress
this._activeStreamContext = {};
this._latestProgressDate = 0;
if (this._closed)
return;
this._progressStatus.finished();
this._downloadStatus = DownloadStatus.Finished;
this._sendProgressDownloadPart();
this.emit("finished");
await this.options.onFinishAsync?.();
}
async _downloadSlice(startChunk, endChunk) {
const getContext = () => this._activeStreamContext[startChunk] ??= { streamBytes: 0, retryingAttempts: 0 };
const fetchState = this.options.fetchStream.withSubState({
chunkSize: this._progress.chunkSize,
startChunk,
endChunk: endChunk,
lastChunkEndsFile: endChunk === Infinity || endChunk === this._progress.chunks.length,
totalSize: this._activePart.size,
url: this._activePart.downloadURL,
rangeSupport: this._activePart.acceptRange,
onProgress: (length) => {
getContext().streamBytes = length;
this._sendProgressDownloadPart();
}
});
fetchState.addListener("retryingOn", () => {
const context = getContext();
context.isRetrying = true;
context.retryingAttempts++;
this._sendProgressDownloadPart();
});
fetchState.addListener("retryingOff", () => {
getContext().isRetrying = false;
});
fetchState.addListener("streamNotRespondingOn", () => {
getContext().isStreamNotResponding = true;
this._sendProgressDownloadPart();
});
fetchState.addListener("streamNotRespondingOff", () => {
getContext().isStreamNotResponding = false;
});
const downloadedPartsSize = this._downloadedPartsSize;
this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS;
const allWrites = new Set();
let lastChunkSize = 0, lastInProgressIndex = startChunk;
await fetchState.fetchChunks((chunks, writePosition, index) => {
if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) {
return;
}
const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunks);
if (writePromise) {
allWrites.add(writePromise);
writePromise.then(() => {
allWrites.delete(writePromise);
});
}
// if content length is 0, we do not know how many chunks we should have
if (this._activePart.size === 0) {
this._progress.chunks.push(ChunkStatus.NOT_STARTED);
}
this._progress.chunks[index] = ChunkStatus.COMPLETE;
lastChunkSize = chunks.reduce((last, current) => last + current.length, 0);
getContext().streamBytes = 0;
void this._saveProgress();
const nextChunk = this._progress.chunks[index + 1];
const shouldReadNext = fetchState.state.endChunk - index > 1; // grater than 1, meaning there is a next chunk
if (shouldReadNext) {
if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) {
return fetchState.close();
}
this._progress.chunks[lastInProgressIndex = index + 1] = ChunkStatus.IN_PROGRESS;
}
});
if (this._progress.chunks[lastInProgressIndex] === ChunkStatus.IN_PROGRESS) {
this._progress.chunks[lastInProgressIndex] = ChunkStatus.NOT_STARTED;
}
// On dynamic content length, we need to adjust the last chunk size
if (this._activePart.size === 0) {
this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize;
this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE);
}
delete this._activeStreamContext[startChunk];
await Promise.all(allWrites);
}
_saveProgress() {
const thisProgress = this._latestProgressDate = Date.now();
this._sendProgressDownloadPart();
if (!this._activePart.acceptRange)
return;
this.emit("save", this._progress);
return withLock(this, "_saveLock", async () => {
if (thisProgress === this._latestProgressDate && !this._closed && this._downloadStatus !== DownloadStatus.Finished) {
await this.options.onSaveProgressAsync?.(this._progress);
}
});
}
_sendProgressDownloadPart() {
if (this._closed)
return;
this.emit("progress", this.status);
}
async pause() {
if (this.options.fetchStream.paused) {
return;
}
this._downloadStatus = DownloadStatus.Paused;
this.options.fetchStream.emit("paused");
await this.options.onPausedAsync?.();
this._sendProgressDownloadPart();
}
resume() {
if (!this.options.fetchStream.paused) {
return;
}
this._downloadStatus = DownloadStatus.Active;
this.options.fetchStream.emit("resumed");
this.emit("resumed");
this._sendProgressDownloadPart();
}
async close() {
if (this._closed)
return;
if (this._downloadStatus !== DownloadStatus.Finished) {
this._progressStatus.finished();
this._downloadStatus = DownloadStatus.Cancelled;
this._sendProgressDownloadPart();
}
this._closed = true;
this._activeProgram?.abort();
await this.options.onCloseAsync?.();
await this.options.writeStream.close();
await this.options.fetchStream.close();
this.emit("closed");
}
finished(comment) {
if (comment) {
this.options.comment = pushComment(comment, this.options.comment);
}
this._downloadStatus = DownloadStatus.Finished;
}
[Symbol.dispose]() {
return this.close();
}
}
//# sourceMappingURL=download-engine-file.js.map