@guoyunhe/downloader/dist/index.js

Download large files with minimum RAM usage. Support tar.gz and zip extraction.

import { createWriteStream } from 'fs';
import { mkdir, mkdtemp } from 'fs/promises';
import http from 'http';
import https from 'https';
import StreamZip from 'node-stream-zip';
import { tmpdir } from 'os';
import { dirname, join } from 'path';
import { PassThrough } from 'stream';
import { x } from 'tar';
import { stripDirectory } from './stripDirectory.js';
export function download(
/** File download URL. Must be public and support GET method. */
url, 
/** Output file (not extract) or folder (extract) path. */
dist, 
/** Extra download options. */
options = {}) {
    const { maxRedirects = 5, extract = false, strip = 0, onProgress } = options;
    return new Promise((resolve, reject) => {
        const get = url.startsWith('https://') ? https.get : http.get;
        const req = get(url, async (res) => {
            if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
                let output;
                let zipFile;
                if (extract &&
                    [
                        'application/tar',
                        'application/tar+gzip',
                        'application/x-tar',
                        'application/x-gzip',
                    ].includes(res.headers['content-type'] || '')) {
                    await mkdir(dist, { recursive: true });
                    output = x({
                        strip,
                        cwd: dist,
                    });
                }
                else if (extract && res.headers['content-type'] === 'application/zip') {
                    const tmpPrefix = join(tmpdir(), 'guoyunhe-downloader');
                    const temp = await mkdtemp(tmpPrefix);
                    zipFile = join(temp, 'archive.zip');
                    output = createWriteStream(zipFile);
                }
                else {
                    await mkdir(dirname(dist), { recursive: true });
                    output = createWriteStream(dist);
                }
                let downloadedBytes = 0;
                const totalBytesHeader = res.headers['content-length'];
                const totalBytes = Number.parseInt(Array.isArray(totalBytesHeader) ? totalBytesHeader[0] : totalBytesHeader || '', 10);
                const normalizedTotalBytes = Number.isFinite(totalBytes) ? totalBytes : null;
                const progressStream = new PassThrough();
                progressStream.on('data', (chunk) => {
                    downloadedBytes += chunk.length;
                    onProgress?.({
                        downloadedBytes,
                        totalBytes: normalizedTotalBytes,
                        percentage: normalizedTotalBytes === null
                            ? null
                            : Math.min((downloadedBytes / normalizedTotalBytes) * 100, 100),
                    });
                });
                output.on('finish', async function () {
                    if (zipFile) {
                        // eslint-disable-next-line new-cap
                        const zip = new StreamZip.async({ file: zipFile });
                        await zip.extract(null, dist);
                        await stripDirectory(dist, strip);
                    }
                    resolve();
                });
                output.on('error', reject);
                progressStream.on('error', reject);
                res.on('error', reject);
                res.pipe(progressStream).pipe(output);
            }
            else if (res.statusCode &&
                res.statusCode >= 300 &&
                res.statusCode < 400 &&
                res.headers.location) {
                if (maxRedirects > 0) {
                    // Follow redirect
                    const newUrl = res.headers.location;
                    download(newUrl, dist, { ...options, maxRedirects: maxRedirects - 1 })
                        .then(resolve)
                        .catch(reject);
                }
                else {
                    // Too many redirects
                    reject(new Error(`Too many redirects when downloading ${url}. Use maxRedirects option to increase the limit.`));
                }
            }
            else {
                reject(new Error(`Failed to download ${url}`));
            }
        });
        req.on('error', reject);
    });
}