@lpb_name/down/src/downloader.js

A Node.js download manager with multi-threading support

import fs from "fs";
import path from "path";
import cliProgress from "cli-progress";
import got from "got";
import http from "http";
import https from "https";

class Timer {
  constructor() {
    this.startTime = 0;
    this.endTime = 0;
  }

  start() {
    this.startTime = Date.now();
  }

  stop() {
    this.endTime = Date.now();
  }

  getDuration() {
    return ((this.endTime - this.startTime) / 1000).toFixed(2);
  }
}

export function createDownloadManager(config) {
  const {
    url,
    output = path.basename(new URL(url).pathname),
    threads = 3,
    timeoutMs = 600000,
  } = config;

  let totalSize = 0;
  let downloadedBytes = 0;
  let totalBar = null;
  let meta = null;
  let metaPath = "";

  const multibar = new cliProgress.MultiBar({
    clearOnComplete: false,
    hideCursor: true,
    format:
      "{threadId} |{bar}| {percentage}% | {value_mb}/{total_mb}MB | {speed}",
    barCompleteChar: "\u2588",
    barIncompleteChar: "\u2591",
    align: "left",
    autopadding: true,
    forceRedraw: false,
    barsize: 30,
    stopOnComplete: true,
  });

  let threadBars = [];
  let mergeBar = null;
  let isCompleted = false;

  const totalTimer = new Timer();
  const downloadTimer = new Timer();
  const mergeTimer = new Timer();

  const agent = {
    http: new http.Agent({ keepAlive: true }),
    https: new https.Agent({ keepAlive: true }),
  };

  function getMetaPath(outPath) {
    return `${outPath}.down.meta.json`;
  }

  async function loadMeta(outPath) {
    const p = getMetaPath(outPath);
    metaPath = p;
    try {
      const raw = await fs.promises.readFile(p, "utf8");
      const m = JSON.parse(raw);
      return m;
    } catch {
      return null;
    }
  }

  async function saveMeta(m) {
    if (!metaPath) metaPath = getMetaPath(output);
    try {
      await fs.promises.writeFile(metaPath, JSON.stringify(m));
    } catch {}
  }

  function planChunks(size, t) {
    const cs = Math.ceil(size / t);
    const chunks = [];
    for (let i = 0; i < t; i++) {
      const start = i * cs;
      const end = i === t - 1 ? size - 1 : start + cs - 1;
      chunks.push({ start, end, downloaded: 0, completed: false });
    }
    return { chunks, chunkSize: cs };
  }

  async function ensurePreallocated(size) {
    try {
      const st = await fs.promises.stat(output).catch(() => null);
      if (!st || st.size !== size) {
        const fd = await fs.promises.open(output, "w");
        await fd.truncate(size);
        await fd.close();
      }
    } catch {}
  }

  async function probeCapabilities() {
    let size = 0;
    let acceptRanges = false;
    try {
      const res = await got.head(url, {
        retry: { limit: 2 },
        timeout: { connect: 10000, response: 10000 },
        agent,
        http2: false,
      });
      size = parseInt(res.headers["content-length"] || "0");
      acceptRanges = (res.headers["accept-ranges"] || "").includes("bytes");
      if (Number.isFinite(size) && size > 0) {
        return { size, acceptRanges };
      }
      throw new Error("NO_LENGTH");
    } catch {
      // Fallback: issue a GET request, read headers then abort immediately
      return await new Promise((resolve) => {
        const req = got.stream(url, {
          method: "GET",
          retry: { limit: 2 },
          timeout: { connect: 10000, response: 10000 },
          agent,
          http2: false,
        });
        req.once("response", (res) => {
          const len = parseInt(res.headers["content-length"] || "0");
          const ranges =
            (res.headers["accept-ranges"] || "").includes("bytes") ||
            res.statusCode === 206;
          try {
            req.destroy();
          } catch {}
          resolve({
            size: Number.isFinite(len) && len > 0 ? len : 0,
            acceptRanges: ranges,
          });
        });
        req.once("error", () => resolve({ size: 0, acceptRanges: false }));
      });
    }
  }

  async function downloadChunk(start, end, index, initialDownloaded = 0) {
    const chunkSize = end - start + 1;
    let threadDownloaded = 0;
    const threadBar = multibar.create(chunkSize, initialDownloaded, {
      threadId: `Thread  ${String(index + 1).padStart(2, " ")}`,
      speed: 0,
      value_mb: (initialDownloaded / (1024 * 1024)).toFixed(2),
      total_mb: (chunkSize / (1024 * 1024)).toFixed(2),
    });
    threadBars[index] = threadBar;
    const startTime = Date.now();
    const effectiveStart = start + initialDownloaded;
    const req = got.stream(url, {
      headers: { Range: `bytes=${effectiveStart}-${end}` },
      retry: { limit: 3 },
      timeout: { connect: timeoutMs, response: timeoutMs },
      agent,
      http2: false,
    });

    const writeStream = fs.createWriteStream(output, {
      flags: "r+",
      start: effectiveStart,
    });

    return new Promise((resolve, reject) => {
      let lastSaved = initialDownloaded;
      req.on("data", (chunk) => {
        if (isCompleted) return;
        downloadedBytes += chunk.length;
        threadDownloaded += chunk.length;
        const currentDownloaded = initialDownloaded + threadDownloaded;
        const elapsed = Date.now() - startTime;
        const speedVal =
          elapsed > 500
            ? (((threadDownloaded / elapsed) * 1000) / 1024 / 1024).toFixed(2)
            : "测量中";
        threadBar.update(currentDownloaded, {
          speed:
            typeof speedVal === "string"
              ? `速度: ${speedVal}`
              : `速度: ${speedVal}MB/s`,
          value_mb: (currentDownloaded / (1024 * 1024)).toFixed(2),
        });
        if (totalBar) {
          totalBar.update(downloadedBytes, {
            value_mb: (downloadedBytes / (1024 * 1024)).toFixed(2),
            total_mb: (totalSize / (1024 * 1024)).toFixed(2),
          });
        }
        if (meta && meta.chunks && meta.chunks[index]) {
          meta.chunks[index].downloaded = currentDownloaded;
          if (currentDownloaded - lastSaved >= 256 * 1024) {
            lastSaved = currentDownloaded;
            saveMeta(meta);
          }
        }
      });

      req.on("error", (error) => {
        try {
          writeStream.destroy();
        } catch {}
        reject(error);
      });

      writeStream.on("error", (error) => {
        try {
          req.destroy();
        } catch {}
        reject(error);
      });
      writeStream.once("finish", () => {
        if (!isCompleted) {
          threadBar.update(chunkSize, {
            speed: "已完成",
            value_mb: (chunkSize / (1024 * 1024)).toFixed(2),
          });
        }
        if (meta && meta.chunks && meta.chunks[index]) {
          meta.chunks[index].completed = true;
          meta.chunks[index].downloaded = chunkSize;
          saveMeta(meta);
        }
        resolve();
      });

      req.pipe(writeStream);
    });
  }

  async function start() {
    try {
      totalTimer.start();
      // 探测能力与文件大小
      const probe = await probeCapabilities();
      totalSize = probe.size || 0;

      // 如不支持 Range 或大小未知，则降级为单线程
      if (!probe.acceptRanges || !totalSize) {
        console.log("服务端不支持分片或无法获取长度，降级为单线程下载");
        downloadTimer.start();
        await new Promise((resolve, reject) => {
          const req = got.stream(url, {
            retry: { limit: 3 },
            timeout: { connect: timeoutMs, response: timeoutMs },
            agent,
            http2: false,
          });
          const ws = fs.createWriteStream(output);
          req.on("data", (chunk) => {
            if (isCompleted) return;
            downloadedBytes += chunk.length;
          });
          req.on("error", (err) => {
            try {
              ws.destroy();
            } catch {}
            reject(err);
          });
          ws.on("error", (err) => {
            try {
              req.destroy();
            } catch {}
            reject(err);
          });
          ws.once("finish", resolve);
          req.pipe(ws);
        });
        downloadTimer.stop();
        mergeTimer.start();
      } else {
        downloadTimer.start();

        meta = await loadMeta(output);
        if (
          !meta ||
          meta.url !== url ||
          meta.totalSize !== totalSize ||
          meta.threads !== threads
        ) {
          const plan = planChunks(totalSize, threads);
          meta = {
            url,
            output,
            totalSize,
            threads,
            chunkSize: plan.chunkSize,
            chunks: plan.chunks,
          };
          await saveMeta(meta);
        }

        await ensurePreallocated(totalSize);

        downloadedBytes = meta.chunks.reduce(
          (sum, c) => sum + (c.downloaded || 0),
          0
        );

        totalBar = multibar.create(
          totalSize,
          downloadedBytes,
          {
            threadId: "总进度",
            value_mb: (downloadedBytes / (1024 * 1024)).toFixed(2),
            total_mb: (totalSize / (1024 * 1024)).toFixed(2),
            speed: "",
          },
          {
            format:
              "{threadId} |{bar}| {percentage}% | {value_mb}/{total_mb}MB",
          }
        );

        const tasks = [];
        for (let i = 0; i < meta.chunks.length; i++) {
          const c = meta.chunks[i];
          if (c.completed) continue;
          const initialDownloaded = c.downloaded || 0;
          tasks.push(downloadChunk(c.start, c.end, i, initialDownloaded));
        }

        const totalChunks = meta.chunks.length;
        const doneChunks = meta.chunks.filter((c) => c.completed).length;
        mergeBar = multibar.create(
          totalChunks,
          doneChunks,
          {
            threadId: "分片完成 ",
            value_mb: doneChunks.toString(),
            total_mb: totalChunks.toString(),
            speed: "",
          },
          {
            format:
              "{threadId} |{bar}| {percentage}% | 已完成: {value_mb}/{total_mb}个分片",
          }
        );

        let completedTasks = doneChunks;
        const downloadPromises = tasks.map((task) =>
          task.then(() => {
            if (!isCompleted) {
              completedTasks++;
              mergeBar.update(completedTasks, {
                value_mb: completedTasks.toString(),
              });
            }
          })
        );

        await Promise.all(downloadPromises);
        downloadTimer.stop();
        mergeTimer.start();
        try {
          await fs.promises.unlink(metaPath);
        } catch {}
      }

      // 标记为完成，停止所有进度条更新
      isCompleted = true;

      mergeTimer.stop();
      totalTimer.stop();

      // 停止进度条显示
      multibar.stop();
      try {
        const st = await fs.promises.stat(output);
        if (totalSize && st.size !== totalSize) {
          console.error(
            `警告: 文件大小不匹配，期望 ${totalSize} 字节，实际 ${st.size} 字节`
          );
        }
      } catch {}

      // 输出完成信息
      console.log(`\n下载完成，耗时: ${downloadTimer.getDuration()}秒`);
      console.log(`合并完成，耗时: ${mergeTimer.getDuration()}秒`);
      console.log(`总耗时: ${totalTimer.getDuration()}秒`);
      console.log(`\n下载完成！`);
    } catch (error) {
      multibar.stop();
      try {
        const st = await fs.promises.stat(output).catch(() => null);
        if (st && st.size === 0) {
          await fs.promises.unlink(output);
        }
      } catch {}
      throw error;
    }
  }

  return {
    start,
  };
}