UNPKG

sharding-downloader

Version:

This project downloads files in the browser,Support large file sharding and IndexedDB(大文件分片下载)

757 lines (753 loc) 23.9 kB
var A = Object.defineProperty; var z = (r, t, e) => t in r ? A(r, t, { enumerable: !0, configurable: !0, writable: !0, value: e }) : r[t] = e; var o = (r, t, e) => z(r, typeof t != "symbol" ? t + "" : t, e); function b(r) { const { action: t, method: e = "POST", data: s = {}, headers: n = {}, responseType: a = "blob", withCredentials: i = !0, onProgress: d = () => { }, onSuccess: h = () => { }, onFail: u = () => { } } = r; let c = new XMLHttpRequest(); return c.responseType = a, c.withCredentials = i, c.open(e, t, !0), "setRequestHeader" in c && Object.entries(n).forEach(([g, m]) => c.setRequestHeader(g, m)), c.addEventListener("timeout", () => u(new Error("Request timed out"), c)), c.addEventListener("progress", d), c.addEventListener( "error", () => u(new Error(`Request failed with status ${c.status}`), c) ), c.addEventListener("readystatechange", () => { if (c.readyState === 4) { if (c.status < 200 || c.status >= 300) { u(new Error(`xhr: status === ${c.status}`), c); return; } h({ data: c.response, status: c.status, headers: { "content-length": c.getResponseHeader("content-length") || "", "content-range": c.getResponseHeader("content-range") || "", "content-disposition": c.getResponseHeader("content-disposition") || "", etag: c.getResponseHeader("etag") || "" } }); } }), c.send(JSON.stringify(s)), { abort() { c.abort(); } }; } const D = (r, t) => toString.call(r).slice(8, -1).toLowerCase() === t, x = (r) => D(r, "object"), f = (r) => x(r) && r !== null, P = (r) => D(r, "blob"), R = (r) => { if (!r) return ""; let t = ""; const e = r.match(/filename="([^"]+)"/i) || r.match(/filename=([^;]+)/i); e && (t = e[1].trim()); const s = r.match(/filename\*=UTF-8''([^;]+)/i); if (s) try { t = decodeURIComponent(s[1].trim()); } catch (n) { console.warn("Failed to decode UTF-8 filename:", n); } return t; }, y = (r) => { if (r === null || r === "" || r === 0) return "0 B"; const e = new Array("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"); let s = 0; const n = parseFloat(String(r)); return s = Math.floor(Math.log(n) / Math.log(1024)), (n / Math.pow(1024, s)).toFixed(2) + " " + e[s]; }, I = async (r) => { if (P(r)) try { const t = await r.text(); return JSON.parse(t); } catch { return r; } if (x(r)) return r; }, k = !!window.indexedDB; async function M(r = 6, t, e) { const s = [], n = /* @__PURE__ */ new Set(); for (const [a, i] of Array.from(t.entries())) { const d = (async () => { try { return await e(i, t); } catch (u) { const c = new Error(`[AsyncPool] Task ${a} failed: ${u.message}`); throw c.cause = u, c.index = a, c; } })(); s[a] = d; const h = d.finally(() => { n.delete(h); }); if (n.add(h), n.size >= r) try { await Promise.race(n); } catch { } } try { await Promise.allSettled(n); const a = await Promise.allSettled(s), i = a.filter((h) => h.status === "rejected").map((h) => h.reason), d = i.map((h) => h.index); if (i.length > 0) throw new AggregateError( i, ` [AsyncPool] 部分任务执行失败 (失败数: ${i.length}/${s.length}); [Error Tasks]: [${d}]} ` ); return a.map((h) => { if (h.status === "fulfilled") return h.value; throw h.reason; }); } catch (a) { throw new Error(a); } } const L = (r, t) => Object.prototype.hasOwnProperty.call(r, t); function S(r, t) { if (!f(t) || !f(r)) return t; const e = Array.isArray(r) ? [...r] : { ...r }; for (const s in t) if (L(t, s)) { const n = t[s], a = e[s]; f(n) && !Array.isArray(n) ? e[s] = S(a, n) : e[s] = n; } return e; } let N = 0; const O = (r = "id") => `${r}-${+/* @__PURE__ */ new Date()}-${N++}`, q = (r, t = 300) => { let e = 0; return function(...s) { let n = +/* @__PURE__ */ new Date(); n - e > t && (e = n, r.apply(this, s)); }; }, B = { storageVersion: 1, storageName: "file_chunks_db", action: "", data: {}, headers: { "content-type": "application/json;charset=UTF-8" }, chunkSize: 1024 * 1024 * 2, threads: 6, customRequest: b, maxRetries: 3, retryInterval: 500, requestSucceed: async (r) => { const t = await I(r); return !(t.code && t.code !== "00000"); }, isPart: !0 }, l = { INIT: "init", READY: "ready", DOWNLOADING: "downloading", DOWNLOADED: "downloaded", CANCELLED: "cancelled", SUCCESS: "success", FAILED: "failed" }, w = { READY: "ready", PENDING: "pending", DOWNLOADING: "downloading", DOWNLOADED: "downloaded", ERROR: "error" }, p = { FAILED: "failed", PROGRESS: "progress", SUCCESS: "success", CHANGE: "change", CANCELLED: "canceled" }; class F { constructor(t, e, s) { /** chunk所在file的实例 */ o(this, "parent"); /** file相关配置 */ o(this, "options"); /** chunk的索引 */ o(this, "index"); /** 下载文件的Obs地址,或者下载标识 */ o(this, "url"); /** 文件大小 */ o(this, "fileSize"); /** 分片大小 */ o(this, "chunkSize"); /** chunk所在文件流的位置起点 */ o(this, "startByte"); /** chunk所在文件流的位置终点 */ o(this, "endByte"); /** chunk大小 */ o(this, "size"); /** chunk状态 */ o(this, "status"); /** 重试次数 */ o(this, "maxRetries"); /** */ o(this, "request"); /** 重试间隔计时器 */ o(this, "timer"); /** 下载进度 */ o(this, "progress"); /** 已下载大小 */ o(this, "loaded"); o(this, "handleProgress", q((t) => { this.loaded = t.loaded, this.progress = Math.min(Math.max(t.loaded / t.total, this.progress), 1), this.status = w.DOWNLOADING, this.parent._updateProgress(); }, 200)); this.parent = e, this.options = s, this.index = t, this.url = s.url, this.fileSize = e.size, this.chunkSize = s.chunkSize, this.startByte = this.chunkSize * this.index, this.endByte = Math.min(this.startByte + this.chunkSize - 1, this.fileSize - 1), this.size = this.endByte - this.startByte + 1, this.status = w.READY, this.maxRetries = s.maxRetries, this.request = null, this.timer = void 0, this.progress = 0, this.loaded = 0; } async send() { return this.status = w.PENDING, new Promise((t, e) => { const s = (i) => { if (!(this.request && this.request.canceled)) { if (this.maxRetries > 0) { this.maxRetries--, this.timer = setTimeout(() => { clearTimeout(this.timer), this.send().then((d) => { t(d); }).catch(e); }, this.options.retryInterval); return; } this.status = w.ERROR, e(i); } }, n = async ({ data: i }) => { if (!await this.options.requestSucceed(i)) return s(new Error("Request failed")); this.setSuccess(), t(i); }, a = (i) => [].includes(i); this.request = this.options.customRequest({ index: this.index, action: `${this.options.action}?index=${this.index}${a(this.index) ? "&error=1" : ""}`, data: { url: this.url, index: this.index, ...this.options.data }, headers: { Range: `bytes=${this.startByte}-${this.endByte}`, ...this.options.headers }, onSuccess: n, onFail: s, onProgress: this.handleProgress }), this.request.canceled = !1; }); } cancel() { this.request && (this.request.canceled = !0, this.request.abort()), this.timer && (clearTimeout(this.timer), this.timer = void 0), this.status = w.ERROR; } setSuccess() { this.status = w.DOWNLOADED, this.loaded = this.size, this.progress = 1; } } class _ { constructor(t, e) { /** 配置 */ o(this, "options"); /** 下载实例 */ o(this, "downloader"); /** 存储store */ o(this, "storage"); /** 文件id */ o(this, "id"); /** 文件名称 */ o(this, "name"); /** 文件大小 */ o(this, "size"); /** 文件唯一标识 */ o(this, "etag"); /** 文件状态 */ o(this, "status"); /** 下载成功后生成的前端地址 */ o(this, "link"); /** 分片大小 */ o(this, "chunkSize"); /** 文件obs地址,或者文件下载标识 */ o(this, "url"); /** 下载接口 */ o(this, "action"); /** 文件下载进度 */ o(this, "progress"); /** 文件已经下载的大小(bit) */ o(this, "loadedSize"); /** 分片chunk数组 */ o(this, "chunks"); /** 总分片数 */ o(this, "totalChunks"); /** 已经下载chunk索引值 */ o(this, "downloaded"); /** 取消获取文件元信息abort */ o(this, "metaAbort"); o(this, "resume", () => { this.status === l.CANCELLED && (this.metaAbort && this.metaAbort.abort(), this.start()); }); this.options = t, this.downloader = e, this.storage = e.storage, this.id = O(), this.name = "", this.size = 0, this.etag = "", this.status = l.INIT, this.link = "", this.chunkSize = t.chunkSize, this.action = t.action, this.url = t.url, this.progress = 0, this.loadedSize = 0, this.chunks = [], this.totalChunks = 0, this.downloaded = /* @__PURE__ */ new Set(), this.metaAbort = null, this.start(); } get renderSize() { return y(this.size); } get renderLoadedSize() { return y(this.loadedSize); } changeStatus(t) { this.status = t, this.downloader.emit(p.CHANGE, this); } createChunks() { this.chunks = [], this.totalChunks = Math.ceil(this.size / this.options.chunkSize); for (let t = 0; t < this.totalChunks; t++) this.chunks.push(new F(t, this, this.options)); } async getMetadata() { const { customRequest: t, action: e, url: s } = this.options; return new Promise((n, a) => { this.metaAbort = t({ action: `${e}?meta=`, data: { url: s, index: -1, ...this.options.data }, headers: { Range: "bytes=0-1", ...this.options.headers }, onSuccess: async ({ headers: i, data: d }) => { if (!await this.options.requestSucceed(d)) throw new Error("Request failed"); try { const h = R(i["content-disposition"]), u = Number(i["content-range"].split("/")[1]), c = i.etag; this.name = h, this.size = u, this.etag = c, n({ name: h, size: u, etag: c }); } catch (h) { console.error("Error parsing response headers:", h), a(new Error("Failed to parse response headers")); } }, onFail: a }); }); } async start() { try { await this.getMetadata(), this.changeStatus(l.READY), this.downloader._addFile(this), this.options.isPart ? this.downloadPart() : this.downloadFull(); } catch (t) { throw console.error("Error:", t), this.downloader.emit(p.FAILED, this), new Error("Failed to start download"); } } async downloadPart() { await this.storage.cleanupExpiredChunks(), console.log("Cleanup expired chunks success"), this.createChunks(); let t = await this.storage.getMetadata(this.etag); (!t || t.totalChunks !== this.totalChunks) && (await this.storage.cleanupFileData(this.etag), t = { fileId: this.etag, fileName: this.name, totalSize: this.size, chunkSize: this.chunkSize, totalChunks: this.totalChunks, action: this.action, url: this.url, updateAt: Date.now(), downloadedChunks: [] }, await this.storage.updateMetadata(this, [])), this.downloaded = new Set(t.downloadedChunks), this.changeStatus(l.DOWNLOADING); try { await this.downloadChunks(), this.changeStatus(l.DOWNLOADED), await this.mergeFilePart(); } catch (e) { throw this.changeStatus(l.FAILED), this.downloader.emit(p.FAILED, this), new Error(e); } } async downloadChunks() { await M(this.options.threads, this.chunks, async (t) => { const e = await this.storage.checkChunk(this.etag, t.index); if (this.downloaded.has(t.index) && e) return t.setSuccess(), this._updateProgress(), t; if (!this.downloaded.has(t.index) && e) return this.downloaded.add(t.index), await this.storage.updateMetadata(this, [...this.downloaded]), t.setSuccess(), this._updateProgress(), t; const s = await t.send(); return await this.storage.saveChunk(this.etag, t.index, t.size, s), this.downloaded.add(t.index), await this.storage.updateMetadata(this, [...this.downloaded]), t; }); } async downloadFull() { const { customRequest: t, action: e, url: s } = this.options; t({ action: e + "?full", data: { url: s, index: -2, ...this.options.data }, headers: this.options.headers, onSuccess: async ({ data: n }) => { if (!await this.options.requestSucceed(n)) throw new Error("Request failed"); this.link = this.generateBlobUrl(n), this.progress = 1, this.loadedSize = this.size, this.changeStatus(l.SUCCESS), this.downloader.emit(p.SUCCESS, this); }, onFail: () => { this.changeStatus(l.FAILED), this.downloader.emit(p.FAILED, this); }, onProgress: (n) => { this.progress = n.loaded / n.total, this.loadedSize = n.loaded, this.changeStatus(l.DOWNLOADING), this.downloader.emit(p.PROGRESS, this); } }); } _updateProgress() { const { loadedSize: t, progress: e } = this.chunks.reduce( ({ loadedSize: s, progress: n }, a) => ({ loadedSize: s + a.loaded, progress: n + a.progress * (a.size / this.size) }), { loadedSize: 0, progress: 0 } ); this.progress = Math.min(1, Math.max(e, this.progress)), this.loadedSize = Math.min(this.size, Math.max(t, this.loadedSize)), this.changeStatus(l.DOWNLOADING), this.downloader.emit(p.PROGRESS, this); } async mergeFilePart() { this.progress = 1, this.loadedSize = this.size; const t = await this.storage.getChunks(this.etag); t.sort((s, n) => s.chunkIndex - n.chunkIndex); const e = new Blob( t.map((s) => s.data), { type: t[0].data.type || "application/octet-stream" } ); this.link = this.generateBlobUrl(e), this.changeStatus(l.SUCCESS), this.downloader.emit(p.SUCCESS, this), this.storage.cleanupFileData(this.etag), t.length = 0; } generateBlobUrl(t) { return window.URL.createObjectURL(t); } revokeBlobUrl() { window.URL.revokeObjectURL(this.link); } cancel() { this.changeStatus(l.CANCELLED), this.downloader.emit(p.CANCELLED, this), this.chunks.forEach((t) => { t.cancel(); }); } pause() { this.cancel(); } retry() { this.status === l.FAILED && this.start(); } } class v { constructor() { o(this, "events"); this.events = /* @__PURE__ */ new Map(); } on(t, e) { if (typeof e != "function") return; const s = this.events.get(t) || []; s.includes(e) || (s.push(e), this.events.set(t, s)); } off(t, e) { var n; if (!this.events.has(t)) return; if (!e) { this.events.set(t, []); return; } const s = (n = this.events.get(t)) == null ? void 0 : n.filter((a) => a !== e); this.events.set(t, s); } emit(t, ...e) { const s = this.events.get(t); s && s.length && s.forEach((n) => n(...e)); } once(t, e) { if (typeof e != "function") return; const s = (...n) => { e(...n), this.off(t, s); }; this.on(t, s); } } const E = 10 * 3600 * 1e3; class T { constructor() { o(this, "store"); this.store = /* @__PURE__ */ new Map(); } checkChunk(t, e) { return new Promise((s) => { const n = this.store.get(t); if (!n) return s(!1); const a = n.get("chunksMap"); if (!a) return s(!1); s(a.has(e) && a.get(e).data); }); } saveChunk(t, e, s, n) { const a = { fileId: t, chunkIndex: e, chunkSize: s, data: n, updateAt: Date.now() }; return new Promise((i) => { const d = this.store.get(t) || /* @__PURE__ */ new Map(), h = d.get("chunksMap") || /* @__PURE__ */ new Map(); h.set(e, a), d.set("chunksMap", h), this.store.set(t, d), i(a); }); } updateMetadata(t, e) { const s = { fileId: t.etag, fileName: t.name, totalSize: t.size, chunkSize: t.chunkSize, totalChunks: t.totalChunks, action: t.action, url: t.url, downloadedChunks: e, updateAt: Date.now() }, n = this.store.get(s.fileId) || /* @__PURE__ */ new Map(); return n.set("metadata", s), this.store.set(s.fileId, n), Promise.resolve(); } getMetadata(t) { return new Promise((e) => { const s = this.store.get(t); if (!s) return e(null); const n = s.get("metadata"); if (!n) return e(null); e(n); }); } getChunks(t) { return new Promise((e) => { const s = this.store.get(t); if (!s) return e([]); const n = s.get("chunksMap"); if (!n) return e([]); const a = Array.from(n.values()); e(a); }); } cleanupFileData(t) { return new Promise((e) => { this.store.delete(t), e(!0); }); } cleanupExpiredChunks() { const t = Date.now(); return new Promise((e) => { for (let s of this.store.values()) { const n = s.get("metadata"), a = n.fileId; if (!s) continue; const i = s.get("chunksMap"); if (!i) { this.store.delete(a); continue; } for (const [d, h] of i.entries()) t - h.updateAt > E && i.delete(d); n && t - n.updateAt > E && this.store.delete(a); } e(!0); }); } close() { this.store.clear(); } } const U = 10 * 3600 * 1e3; class $ { constructor(t = 1, e = "file_chunks_db") { o(this, "dbName"); o(this, "version"); o(this, "db"); this.dbName = e, this.version = t, this.db = null; } init() { return new Promise((t, e) => { const s = window.indexedDB.open(this.dbName, this.version); s.onupgradeneeded = (n) => { const a = n.target.result; if (!a.objectStoreNames.contains("chunks")) { const i = a.createObjectStore("chunks", { keyPath: ["fileId", "chunkIndex"] }); i.createIndex("fileId", "fileId", { unique: !1 }), i.createIndex("updateAt", "updateAt", { unique: !1 }); } a.objectStoreNames.contains("metadata") || a.createObjectStore("metadata", { keyPath: "fileId" }).createIndex("updateAt", "updateAt", { unique: !1 }); }, s.onsuccess = (n) => { this.db = n.target.result, t(this.db); }, s.onerror = (n) => { e(n.target.error); }; }); } async transaction(t, e = "readonly") { return await this.init(), this.db.transaction([t], e).objectStore(t); } async checkChunk(t, e) { const s = await this.transaction("chunks", "readonly"); return new Promise((n, a) => { const i = s.get([t, e]); i.onsuccess = () => { i.result && i.result.data ? (console.log(`✓ ${e}`), n(!0)) : (console.log(`✗ ${e} ---`), n(!1)); }, i.onerror = () => a(i.error); }); } async saveChunk(t, e, s, n) { const a = { fileId: t, chunkIndex: e, chunkSize: s, data: n, updateAt: Date.now() }, i = await this.transaction("chunks", "readwrite"); return new Promise((d, h) => { const u = i.put(a); u.onsuccess = () => { console.log(`${e} Put successful -----`), d(!0); }, u.onerror = () => h(u.error); }); } async updateMetadata(t, e) { const s = { fileId: t.etag, fileName: t.name, totalSize: t.size, chunkSize: t.chunkSize, totalChunks: t.totalChunks, action: t.action, url: t.url, downloadedChunks: e, updateAt: Date.now() }, n = await this.transaction("metadata", "readwrite"); return new Promise((a, i) => { const d = n.put(s); d.onsuccess = () => { a(d.result); }, d.onerror = (h) => i(h.target.error); }); } async getMetadata(t) { const e = await this.transaction("metadata"); return new Promise((s, n) => { const a = e.get(t); a.onsuccess = () => { s(a.result); }, a.onerror = (i) => n(i.target.error); }); } async getChunks(t) { const e = await this.transaction("chunks"); return new Promise((s, n) => { const i = e.index("fileId").getAll(t); i.onsuccess = () => s(i.result), i.onerror = (d) => n(d.target.error); }); } async cleanupFileData(t) { await this.init(); const e = this.db.transaction(["chunks", "metadata"], "readwrite"), n = e.objectStore("chunks").index("fileId"), a = IDBKeyRange.only(t); return n.openCursor(a).onsuccess = (d) => { const h = d.target.result; h && (h.delete(), h.continue()); }, e.objectStore("metadata").delete(t), new Promise((d, h) => { e.oncomplete = () => d(!0), e.onerror = () => h(e.error); }); } async cleanupExpiredChunks() { const t = Date.now(); await this.init(); const e = this.db.transaction(["chunks", "metadata"], "readwrite"), s = IDBKeyRange.upperBound(t - U), i = e.objectStore("metadata").index("updateAt").openCursor(s); await new Promise((c) => { i.onsuccess = (g) => { const m = g.target.result; m && (m.delete(), m.continue()), c(!0); }; }); const u = e.objectStore("chunks").index("updateAt"); return u.openCursor(s).onsuccess = (c) => { const g = c.target.result; g && (g.delete(), g.continue()); }, new Promise((c, g) => { e.oncomplete = () => { c(!0); }, e.onerror = () => g(e.error); }); } close() { this.db && (this.db.close(), this.db = null); } } class C { constructor(t = 1, e = "file_chunks_db") { o(this, "_store"); o(this, "type"); this._store = k ? new $(t, e) : new T(), this.type = k ? "IndexedDB" : "Memory"; } async checkChunk(t, e) { return this._store.checkChunk(t, e); } async saveChunk(t, e, s, n) { return this._store.saveChunk(t, e, s, n); } async updateMetadata(t, e) { return this._store.updateMetadata(t, e); } async getMetadata(t) { return this._store.getMetadata(t); } async getChunks(t) { return this._store.getChunks(t); } async cleanupFileData(t) { return this._store.cleanupFileData(t); } async cleanupExpiredChunks() { return this._store.cleanupExpiredChunks(); } close() { this._store && this._store.close(); } } class G { constructor(t) { o(this, "options"); o(this, "event"); o(this, "storage"); o(this, "fileList"); this.options = S(B, t), this.event = new v(), this.storage = new C(this.options.storageVersion, this.options.storageName), this.fileList = []; } on(t, e) { this.event.on(t, e); } emit(t, ...e) { this.event.emit(t, ...e, this.fileList); } setOption(t) { this.options = S(this.options, t), this.storage = new C(t.storageVersion, t.storageName); } async start(t) { if (typeof t != "string" || !t.trim()) throw this.emit(p.FAILED, null, this.fileList), new Error("A valid URL is required"); if (!this.options.customRequest && !this.options.action) throw new Error("Config action is required"); new _({ ...this.options, url: t }, this); } _addFile(t) { this.fileList.some((e) => e.id === t.id) || this.fileList.push(t); } } export { p as Callbacks, G as Downloader, l as FileStatus };