@guoyunhe/downloader/dist/esm/index.js

Download large files with minimum RAM usage. Support tar.gz and zip extraction.

// src/index.ts
import { createWriteStream } from "fs";
import { mkdir as mkdir2, mkdtemp } from "fs/promises";
import http from "http";
import https from "https";
import StreamZip from "node-stream-zip";
import { tmpdir } from "os";
import { dirname, join as join2 } from "path";
import tar from "tar";

// src/stripDirectory.ts
import { mkdir, readdir, rename, rm } from "fs/promises";
import { join } from "path";
async function stripDirectory(dir, strip = 1) {
  if (strip < 1)
    return;
  const items = await scanDirectory(dir, strip + 1);
  const tmpDir = dir + "__old__";
  await rename(dir, tmpDir);
  await mkdir(dir);
  await Promise.all(
    Object.entries(items).map(async ([from, to]) => {
      rename(join(tmpDir, from), join(dir, to));
    })
  );
  await rm(tmpDir, { recursive: true });
}
async function scanDirectory(dir, depth = 1, prefix = "") {
  const result = {};
  if (depth < 1)
    return result;
  const items = await readdir(dir, { withFileTypes: true });
  for (let i = 0; i < items.length; i++) {
    const item = items[i];
    if (item.isDirectory() && depth > 1) {
      Object.assign(
        result,
        await scanDirectory(join(dir, item.name), depth - 1, join(prefix, item.name))
      );
    } else {
      result[join(prefix, item.name)] = item.name;
    }
  }
  return result;
}

// src/index.ts
function download(url, dist, options = {}) {
  const { maxRedirects = 5, extract = false, strip = 0 } = options;
  return new Promise((resolve, reject) => {
    const get = url.startsWith("https://") ? https.get : http.get;
    get(url, async (res) => {
      if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
        let output;
        let zipFile;
        if (extract && [
          "application/tar",
          "application/tar+gzip",
          "application/x-tar",
          "application/x-gzip"
        ].includes(res.headers["content-type"] || "")) {
          await mkdir2(dist, { recursive: true });
          output = tar.x({
            strip,
            cwd: dist
          });
        } else if (extract && res.headers["content-type"] === "application/zip") {
          const tmpPrefix = join2(tmpdir(), "guoyunhe-downloader");
          const temp = await mkdtemp(tmpPrefix);
          zipFile = join2(temp, "archive.zip");
          output = createWriteStream(zipFile);
        } else {
          await mkdir2(dirname(dist), { recursive: true });
          output = createWriteStream(dist);
        }
        res.pipe(output, { end: false });
        res.on("end", async function() {
          output.end();
          if (zipFile) {
            const zip = new StreamZip.async({ file: zipFile });
            await zip.extract(null, dist);
            await stripDirectory(dist, strip);
          }
          resolve();
        });
      } else if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        if (maxRedirects > 0) {
          const newUrl = res.headers.location;
          download(newUrl, dist, { ...options, maxRedirects: maxRedirects - 1 }).then(resolve).catch(reject);
        } else {
          reject(
            new Error(
              `Too many redirects when downloading ${url}. Use maxRedirects option to increase the limit.`
            )
          );
        }
      } else {
        reject(new Error("Failed to download " + url));
      }
    });
  });
}
export {
  download
};