ipull
Version:
The only file downloader you'll ever need. For node.js and the browser, CLI and library for fast and reliable file downloads.
147 lines • 5.17 kB
JavaScript
import fs from "fs/promises";
import fsExtra from "fs-extra";
import retry from "async-retry";
import { withLock } from "lifecycle-utils";
import BaseDownloadEngineWriteStream from "./base-download-engine-write-stream.js";
import WriterIsClosedError from "./errors/writer-is-closed-error.js";
import { BytesWriteDebounce } from "./utils/BytesWriteDebounce.js";
const DEFAULT_OPTIONS = {
mode: "r+",
debounceWrite: {
maxTime: 1000 * 5, // 5 seconds
maxSize: 1024 * 1024 * 2 // 2 MB
}
};
const MAX_AUTO_DEBOUNCE_SIZE = 1024 * 1024 * 100; // 100 MB
const AUTO_DEBOUNCE_SIZE_PERCENT = 0.05;
const MAX_META_SIZE = 10485760; // 10 MB
const NOT_ENOUGH_SPACE_ERROR_CODE = "ENOSPC";
export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineWriteStream {
path;
finalPath;
_fd = null;
_fileWriteFinished = false;
_writeDebounce;
_fileSize = 0;
options;
autoDebounceMaxSize = false;
constructor(path, finalPath, options = {}) {
super();
this.path = path;
this.finalPath = finalPath;
this.autoDebounceMaxSize = !options.debounceWrite?.maxSize;
const optionsWithDefaults = this.options = {
...DEFAULT_OPTIONS,
...options,
debounceWrite: {
...DEFAULT_OPTIONS.debounceWrite,
...options.debounceWrite
}
};
this._writeDebounce = new BytesWriteDebounce({
...optionsWithDefaults.debounceWrite,
writev: (cursor, buffers) => this._writeWithoutDebounce(cursor, buffers)
});
}
get fileSize() {
return this._fileSize;
}
set fileSize(value) {
this._fileSize = value;
if (this.autoDebounceMaxSize) {
this.options.debounceWrite.maxSize = Math.max(Math.min(value * AUTO_DEBOUNCE_SIZE_PERCENT, MAX_AUTO_DEBOUNCE_SIZE), DEFAULT_OPTIONS.debounceWrite.maxSize);
}
}
async _ensureFileOpen() {
return await withLock(this, "_lock", async () => {
if (this._fd) {
return this._fd;
}
return await retry(async () => {
await fsExtra.ensureFile(this.path);
return this._fd = await fs.open(this.path, this.options.mode);
}, this.options.retry);
});
}
async write(cursor, buffers) {
await this._writeDebounce.addChunk(cursor, buffers);
}
async _writeWithoutDebounce(cursor, buffers) {
let throwError = false;
await retry(async () => {
try {
return await this._writeWithoutRetry(cursor, buffers);
}
catch (error) {
if (error?.code === NOT_ENOUGH_SPACE_ERROR_CODE) {
throwError = error;
return;
}
throw error;
}
}, this.options.retry);
if (throwError) {
throw throwError;
}
}
async ensureBytesSynced() {
await this._writeDebounce.writeAllAndFinish();
}
async ftruncate(size = this._fileSize) {
await this.ensureBytesSynced();
this._fileWriteFinished = true;
await retry(async () => {
const fd = await this._ensureFileOpen();
await fd.truncate(size);
}, this.options.retry);
}
async saveMetadataAfterFile(data) {
if (this._fileWriteFinished) {
throw new WriterIsClosedError();
}
const jsonString = JSON.stringify(data);
const encoder = new TextEncoder();
const uint8Array = encoder.encode(jsonString);
await this.write(this._fileSize, [uint8Array]);
}
async loadMetadataAfterFileWithoutRetry() {
if (!await fsExtra.pathExists(this.path)) {
return;
}
const fd = await this._ensureFileOpen();
try {
const state = await fd.stat();
const metadataSize = state.size - this._fileSize;
if (metadataSize <= 0 || metadataSize >= MAX_META_SIZE) {
if (this._fileSize > 0 && state.size > this._fileSize) {
await this.ftruncate();
}
return;
}
const metadataBuffer = Buffer.alloc(metadataSize);
await fd.read(metadataBuffer, 0, metadataSize, this._fileSize);
const decoder = new TextDecoder();
const metadataString = decoder.decode(metadataBuffer);
try {
return JSON.parse(metadataString);
}
catch { }
}
finally {
this._fd = null;
await fd.close();
}
}
async _writeWithoutRetry(cursor, buffers) {
return await withLock(this, "lockWriteOperation", async () => {
const fd = await this._ensureFileOpen();
const { bytesWritten } = await fd.writev(buffers, cursor);
return bytesWritten;
});
}
async close() {
await this._fd?.close();
this._fd = null;
}
}
//# sourceMappingURL=download-engine-write-stream-nodejs.js.map