sharding-downloader
Version:
This project downloads files in the browser,Support large file sharding and IndexedDB(大文件分片下载)
757 lines (753 loc) • 23.9 kB
JavaScript
var A = Object.defineProperty;
var z = (r, t, e) => t in r ? A(r, t, { enumerable: !0, configurable: !0, writable: !0, value: e }) : r[t] = e;
var o = (r, t, e) => z(r, typeof t != "symbol" ? t + "" : t, e);
function b(r) {
const {
action: t,
method: e = "POST",
data: s = {},
headers: n = {},
responseType: a = "blob",
withCredentials: i = !0,
onProgress: d = () => {
},
onSuccess: h = () => {
},
onFail: u = () => {
}
} = r;
let c = new XMLHttpRequest();
return c.responseType = a, c.withCredentials = i, c.open(e, t, !0), "setRequestHeader" in c && Object.entries(n).forEach(([g, m]) => c.setRequestHeader(g, m)), c.addEventListener("timeout", () => u(new Error("Request timed out"), c)), c.addEventListener("progress", d), c.addEventListener(
"error",
() => u(new Error(`Request failed with status ${c.status}`), c)
), c.addEventListener("readystatechange", () => {
if (c.readyState === 4) {
if (c.status < 200 || c.status >= 300) {
u(new Error(`xhr: status === ${c.status}`), c);
return;
}
h({
data: c.response,
status: c.status,
headers: {
"content-length": c.getResponseHeader("content-length") || "",
"content-range": c.getResponseHeader("content-range") || "",
"content-disposition": c.getResponseHeader("content-disposition") || "",
etag: c.getResponseHeader("etag") || ""
}
});
}
}), c.send(JSON.stringify(s)), {
abort() {
c.abort();
}
};
}
const D = (r, t) => toString.call(r).slice(8, -1).toLowerCase() === t, x = (r) => D(r, "object"), f = (r) => x(r) && r !== null, P = (r) => D(r, "blob"), R = (r) => {
if (!r) return "";
let t = "";
const e = r.match(/filename="([^"]+)"/i) || r.match(/filename=([^;]+)/i);
e && (t = e[1].trim());
const s = r.match(/filename\*=UTF-8''([^;]+)/i);
if (s)
try {
t = decodeURIComponent(s[1].trim());
} catch (n) {
console.warn("Failed to decode UTF-8 filename:", n);
}
return t;
}, y = (r) => {
if (r === null || r === "" || r === 0)
return "0 B";
const e = new Array("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB");
let s = 0;
const n = parseFloat(String(r));
return s = Math.floor(Math.log(n) / Math.log(1024)), (n / Math.pow(1024, s)).toFixed(2) + " " + e[s];
}, I = async (r) => {
if (P(r))
try {
const t = await r.text();
return JSON.parse(t);
} catch {
return r;
}
if (x(r))
return r;
}, k = !!window.indexedDB;
async function M(r = 6, t, e) {
const s = [], n = /* @__PURE__ */ new Set();
for (const [a, i] of Array.from(t.entries())) {
const d = (async () => {
try {
return await e(i, t);
} catch (u) {
const c = new Error(`[AsyncPool] Task ${a} failed: ${u.message}`);
throw c.cause = u, c.index = a, c;
}
})();
s[a] = d;
const h = d.finally(() => {
n.delete(h);
});
if (n.add(h), n.size >= r)
try {
await Promise.race(n);
} catch {
}
}
try {
await Promise.allSettled(n);
const a = await Promise.allSettled(s), i = a.filter((h) => h.status === "rejected").map((h) => h.reason), d = i.map((h) => h.index);
if (i.length > 0)
throw new AggregateError(
i,
`
[AsyncPool] 部分任务执行失败 (失败数: ${i.length}/${s.length});
[Error Tasks]: [${d}]}
`
);
return a.map((h) => {
if (h.status === "fulfilled") return h.value;
throw h.reason;
});
} catch (a) {
throw new Error(a);
}
}
const L = (r, t) => Object.prototype.hasOwnProperty.call(r, t);
function S(r, t) {
if (!f(t) || !f(r))
return t;
const e = Array.isArray(r) ? [...r] : { ...r };
for (const s in t)
if (L(t, s)) {
const n = t[s], a = e[s];
f(n) && !Array.isArray(n) ? e[s] = S(a, n) : e[s] = n;
}
return e;
}
let N = 0;
const O = (r = "id") => `${r}-${+/* @__PURE__ */ new Date()}-${N++}`, q = (r, t = 300) => {
let e = 0;
return function(...s) {
let n = +/* @__PURE__ */ new Date();
n - e > t && (e = n, r.apply(this, s));
};
}, B = {
storageVersion: 1,
storageName: "file_chunks_db",
action: "",
data: {},
headers: {
"content-type": "application/json;charset=UTF-8"
},
chunkSize: 1024 * 1024 * 2,
threads: 6,
customRequest: b,
maxRetries: 3,
retryInterval: 500,
requestSucceed: async (r) => {
const t = await I(r);
return !(t.code && t.code !== "00000");
},
isPart: !0
}, l = {
INIT: "init",
READY: "ready",
DOWNLOADING: "downloading",
DOWNLOADED: "downloaded",
CANCELLED: "cancelled",
SUCCESS: "success",
FAILED: "failed"
}, w = {
READY: "ready",
PENDING: "pending",
DOWNLOADING: "downloading",
DOWNLOADED: "downloaded",
ERROR: "error"
}, p = {
FAILED: "failed",
PROGRESS: "progress",
SUCCESS: "success",
CHANGE: "change",
CANCELLED: "canceled"
};
class F {
constructor(t, e, s) {
/** chunk所在file的实例 */
o(this, "parent");
/** file相关配置 */
o(this, "options");
/** chunk的索引 */
o(this, "index");
/** 下载文件的Obs地址,或者下载标识 */
o(this, "url");
/** 文件大小 */
o(this, "fileSize");
/** 分片大小 */
o(this, "chunkSize");
/** chunk所在文件流的位置起点 */
o(this, "startByte");
/** chunk所在文件流的位置终点 */
o(this, "endByte");
/** chunk大小 */
o(this, "size");
/** chunk状态 */
o(this, "status");
/** 重试次数 */
o(this, "maxRetries");
/** */
o(this, "request");
/** 重试间隔计时器 */
o(this, "timer");
/** 下载进度 */
o(this, "progress");
/** 已下载大小 */
o(this, "loaded");
o(this, "handleProgress", q((t) => {
this.loaded = t.loaded, this.progress = Math.min(Math.max(t.loaded / t.total, this.progress), 1), this.status = w.DOWNLOADING, this.parent._updateProgress();
}, 200));
this.parent = e, this.options = s, this.index = t, this.url = s.url, this.fileSize = e.size, this.chunkSize = s.chunkSize, this.startByte = this.chunkSize * this.index, this.endByte = Math.min(this.startByte + this.chunkSize - 1, this.fileSize - 1), this.size = this.endByte - this.startByte + 1, this.status = w.READY, this.maxRetries = s.maxRetries, this.request = null, this.timer = void 0, this.progress = 0, this.loaded = 0;
}
async send() {
return this.status = w.PENDING, new Promise((t, e) => {
const s = (i) => {
if (!(this.request && this.request.canceled)) {
if (this.maxRetries > 0) {
this.maxRetries--, this.timer = setTimeout(() => {
clearTimeout(this.timer), this.send().then((d) => {
t(d);
}).catch(e);
}, this.options.retryInterval);
return;
}
this.status = w.ERROR, e(i);
}
}, n = async ({ data: i }) => {
if (!await this.options.requestSucceed(i))
return s(new Error("Request failed"));
this.setSuccess(), t(i);
}, a = (i) => [].includes(i);
this.request = this.options.customRequest({
index: this.index,
action: `${this.options.action}?index=${this.index}${a(this.index) ? "&error=1" : ""}`,
data: {
url: this.url,
index: this.index,
...this.options.data
},
headers: {
Range: `bytes=${this.startByte}-${this.endByte}`,
...this.options.headers
},
onSuccess: n,
onFail: s,
onProgress: this.handleProgress
}), this.request.canceled = !1;
});
}
cancel() {
this.request && (this.request.canceled = !0, this.request.abort()), this.timer && (clearTimeout(this.timer), this.timer = void 0), this.status = w.ERROR;
}
setSuccess() {
this.status = w.DOWNLOADED, this.loaded = this.size, this.progress = 1;
}
}
class _ {
constructor(t, e) {
/** 配置 */
o(this, "options");
/** 下载实例 */
o(this, "downloader");
/** 存储store */
o(this, "storage");
/** 文件id */
o(this, "id");
/** 文件名称 */
o(this, "name");
/** 文件大小 */
o(this, "size");
/** 文件唯一标识 */
o(this, "etag");
/** 文件状态 */
o(this, "status");
/** 下载成功后生成的前端地址 */
o(this, "link");
/** 分片大小 */
o(this, "chunkSize");
/** 文件obs地址,或者文件下载标识 */
o(this, "url");
/** 下载接口 */
o(this, "action");
/** 文件下载进度 */
o(this, "progress");
/** 文件已经下载的大小(bit) */
o(this, "loadedSize");
/** 分片chunk数组 */
o(this, "chunks");
/** 总分片数 */
o(this, "totalChunks");
/** 已经下载chunk索引值 */
o(this, "downloaded");
/** 取消获取文件元信息abort */
o(this, "metaAbort");
o(this, "resume", () => {
this.status === l.CANCELLED && (this.metaAbort && this.metaAbort.abort(), this.start());
});
this.options = t, this.downloader = e, this.storage = e.storage, this.id = O(), this.name = "", this.size = 0, this.etag = "", this.status = l.INIT, this.link = "", this.chunkSize = t.chunkSize, this.action = t.action, this.url = t.url, this.progress = 0, this.loadedSize = 0, this.chunks = [], this.totalChunks = 0, this.downloaded = /* @__PURE__ */ new Set(), this.metaAbort = null, this.start();
}
get renderSize() {
return y(this.size);
}
get renderLoadedSize() {
return y(this.loadedSize);
}
changeStatus(t) {
this.status = t, this.downloader.emit(p.CHANGE, this);
}
createChunks() {
this.chunks = [], this.totalChunks = Math.ceil(this.size / this.options.chunkSize);
for (let t = 0; t < this.totalChunks; t++)
this.chunks.push(new F(t, this, this.options));
}
async getMetadata() {
const { customRequest: t, action: e, url: s } = this.options;
return new Promise((n, a) => {
this.metaAbort = t({
action: `${e}?meta=`,
data: {
url: s,
index: -1,
...this.options.data
},
headers: {
Range: "bytes=0-1",
...this.options.headers
},
onSuccess: async ({ headers: i, data: d }) => {
if (!await this.options.requestSucceed(d))
throw new Error("Request failed");
try {
const h = R(i["content-disposition"]), u = Number(i["content-range"].split("/")[1]), c = i.etag;
this.name = h, this.size = u, this.etag = c, n({ name: h, size: u, etag: c });
} catch (h) {
console.error("Error parsing response headers:", h), a(new Error("Failed to parse response headers"));
}
},
onFail: a
});
});
}
async start() {
try {
await this.getMetadata(), this.changeStatus(l.READY), this.downloader._addFile(this), this.options.isPart ? this.downloadPart() : this.downloadFull();
} catch (t) {
throw console.error("Error:", t), this.downloader.emit(p.FAILED, this), new Error("Failed to start download");
}
}
async downloadPart() {
await this.storage.cleanupExpiredChunks(), console.log("Cleanup expired chunks success"), this.createChunks();
let t = await this.storage.getMetadata(this.etag);
(!t || t.totalChunks !== this.totalChunks) && (await this.storage.cleanupFileData(this.etag), t = {
fileId: this.etag,
fileName: this.name,
totalSize: this.size,
chunkSize: this.chunkSize,
totalChunks: this.totalChunks,
action: this.action,
url: this.url,
updateAt: Date.now(),
downloadedChunks: []
}, await this.storage.updateMetadata(this, [])), this.downloaded = new Set(t.downloadedChunks), this.changeStatus(l.DOWNLOADING);
try {
await this.downloadChunks(), this.changeStatus(l.DOWNLOADED), await this.mergeFilePart();
} catch (e) {
throw this.changeStatus(l.FAILED), this.downloader.emit(p.FAILED, this), new Error(e);
}
}
async downloadChunks() {
await M(this.options.threads, this.chunks, async (t) => {
const e = await this.storage.checkChunk(this.etag, t.index);
if (this.downloaded.has(t.index) && e)
return t.setSuccess(), this._updateProgress(), t;
if (!this.downloaded.has(t.index) && e)
return this.downloaded.add(t.index), await this.storage.updateMetadata(this, [...this.downloaded]), t.setSuccess(), this._updateProgress(), t;
const s = await t.send();
return await this.storage.saveChunk(this.etag, t.index, t.size, s), this.downloaded.add(t.index), await this.storage.updateMetadata(this, [...this.downloaded]), t;
});
}
async downloadFull() {
const { customRequest: t, action: e, url: s } = this.options;
t({
action: e + "?full",
data: {
url: s,
index: -2,
...this.options.data
},
headers: this.options.headers,
onSuccess: async ({ data: n }) => {
if (!await this.options.requestSucceed(n))
throw new Error("Request failed");
this.link = this.generateBlobUrl(n), this.progress = 1, this.loadedSize = this.size, this.changeStatus(l.SUCCESS), this.downloader.emit(p.SUCCESS, this);
},
onFail: () => {
this.changeStatus(l.FAILED), this.downloader.emit(p.FAILED, this);
},
onProgress: (n) => {
this.progress = n.loaded / n.total, this.loadedSize = n.loaded, this.changeStatus(l.DOWNLOADING), this.downloader.emit(p.PROGRESS, this);
}
});
}
_updateProgress() {
const { loadedSize: t, progress: e } = this.chunks.reduce(
({ loadedSize: s, progress: n }, a) => ({
loadedSize: s + a.loaded,
progress: n + a.progress * (a.size / this.size)
}),
{ loadedSize: 0, progress: 0 }
);
this.progress = Math.min(1, Math.max(e, this.progress)), this.loadedSize = Math.min(this.size, Math.max(t, this.loadedSize)), this.changeStatus(l.DOWNLOADING), this.downloader.emit(p.PROGRESS, this);
}
async mergeFilePart() {
this.progress = 1, this.loadedSize = this.size;
const t = await this.storage.getChunks(this.etag);
t.sort((s, n) => s.chunkIndex - n.chunkIndex);
const e = new Blob(
t.map((s) => s.data),
{ type: t[0].data.type || "application/octet-stream" }
);
this.link = this.generateBlobUrl(e), this.changeStatus(l.SUCCESS), this.downloader.emit(p.SUCCESS, this), this.storage.cleanupFileData(this.etag), t.length = 0;
}
generateBlobUrl(t) {
return window.URL.createObjectURL(t);
}
revokeBlobUrl() {
window.URL.revokeObjectURL(this.link);
}
cancel() {
this.changeStatus(l.CANCELLED), this.downloader.emit(p.CANCELLED, this), this.chunks.forEach((t) => {
t.cancel();
});
}
pause() {
this.cancel();
}
retry() {
this.status === l.FAILED && this.start();
}
}
class v {
constructor() {
o(this, "events");
this.events = /* @__PURE__ */ new Map();
}
on(t, e) {
if (typeof e != "function") return;
const s = this.events.get(t) || [];
s.includes(e) || (s.push(e), this.events.set(t, s));
}
off(t, e) {
var n;
if (!this.events.has(t)) return;
if (!e) {
this.events.set(t, []);
return;
}
const s = (n = this.events.get(t)) == null ? void 0 : n.filter((a) => a !== e);
this.events.set(t, s);
}
emit(t, ...e) {
const s = this.events.get(t);
s && s.length && s.forEach((n) => n(...e));
}
once(t, e) {
if (typeof e != "function") return;
const s = (...n) => {
e(...n), this.off(t, s);
};
this.on(t, s);
}
}
const E = 10 * 3600 * 1e3;
class T {
constructor() {
o(this, "store");
this.store = /* @__PURE__ */ new Map();
}
checkChunk(t, e) {
return new Promise((s) => {
const n = this.store.get(t);
if (!n)
return s(!1);
const a = n.get("chunksMap");
if (!a)
return s(!1);
s(a.has(e) && a.get(e).data);
});
}
saveChunk(t, e, s, n) {
const a = {
fileId: t,
chunkIndex: e,
chunkSize: s,
data: n,
updateAt: Date.now()
};
return new Promise((i) => {
const d = this.store.get(t) || /* @__PURE__ */ new Map(), h = d.get("chunksMap") || /* @__PURE__ */ new Map();
h.set(e, a), d.set("chunksMap", h), this.store.set(t, d), i(a);
});
}
updateMetadata(t, e) {
const s = {
fileId: t.etag,
fileName: t.name,
totalSize: t.size,
chunkSize: t.chunkSize,
totalChunks: t.totalChunks,
action: t.action,
url: t.url,
downloadedChunks: e,
updateAt: Date.now()
}, n = this.store.get(s.fileId) || /* @__PURE__ */ new Map();
return n.set("metadata", s), this.store.set(s.fileId, n), Promise.resolve();
}
getMetadata(t) {
return new Promise((e) => {
const s = this.store.get(t);
if (!s)
return e(null);
const n = s.get("metadata");
if (!n)
return e(null);
e(n);
});
}
getChunks(t) {
return new Promise((e) => {
const s = this.store.get(t);
if (!s)
return e([]);
const n = s.get("chunksMap");
if (!n)
return e([]);
const a = Array.from(n.values());
e(a);
});
}
cleanupFileData(t) {
return new Promise((e) => {
this.store.delete(t), e(!0);
});
}
cleanupExpiredChunks() {
const t = Date.now();
return new Promise((e) => {
for (let s of this.store.values()) {
const n = s.get("metadata"), a = n.fileId;
if (!s)
continue;
const i = s.get("chunksMap");
if (!i) {
this.store.delete(a);
continue;
}
for (const [d, h] of i.entries())
t - h.updateAt > E && i.delete(d);
n && t - n.updateAt > E && this.store.delete(a);
}
e(!0);
});
}
close() {
this.store.clear();
}
}
const U = 10 * 3600 * 1e3;
class $ {
constructor(t = 1, e = "file_chunks_db") {
o(this, "dbName");
o(this, "version");
o(this, "db");
this.dbName = e, this.version = t, this.db = null;
}
init() {
return new Promise((t, e) => {
const s = window.indexedDB.open(this.dbName, this.version);
s.onupgradeneeded = (n) => {
const a = n.target.result;
if (!a.objectStoreNames.contains("chunks")) {
const i = a.createObjectStore("chunks", {
keyPath: ["fileId", "chunkIndex"]
});
i.createIndex("fileId", "fileId", { unique: !1 }), i.createIndex("updateAt", "updateAt", { unique: !1 });
}
a.objectStoreNames.contains("metadata") || a.createObjectStore("metadata", { keyPath: "fileId" }).createIndex("updateAt", "updateAt", { unique: !1 });
}, s.onsuccess = (n) => {
this.db = n.target.result, t(this.db);
}, s.onerror = (n) => {
e(n.target.error);
};
});
}
async transaction(t, e = "readonly") {
return await this.init(), this.db.transaction([t], e).objectStore(t);
}
async checkChunk(t, e) {
const s = await this.transaction("chunks", "readonly");
return new Promise((n, a) => {
const i = s.get([t, e]);
i.onsuccess = () => {
i.result && i.result.data ? (console.log(`✓ ${e}`), n(!0)) : (console.log(`✗ ${e} ---`), n(!1));
}, i.onerror = () => a(i.error);
});
}
async saveChunk(t, e, s, n) {
const a = {
fileId: t,
chunkIndex: e,
chunkSize: s,
data: n,
updateAt: Date.now()
}, i = await this.transaction("chunks", "readwrite");
return new Promise((d, h) => {
const u = i.put(a);
u.onsuccess = () => {
console.log(`${e} Put successful -----`), d(!0);
}, u.onerror = () => h(u.error);
});
}
async updateMetadata(t, e) {
const s = {
fileId: t.etag,
fileName: t.name,
totalSize: t.size,
chunkSize: t.chunkSize,
totalChunks: t.totalChunks,
action: t.action,
url: t.url,
downloadedChunks: e,
updateAt: Date.now()
}, n = await this.transaction("metadata", "readwrite");
return new Promise((a, i) => {
const d = n.put(s);
d.onsuccess = () => {
a(d.result);
}, d.onerror = (h) => i(h.target.error);
});
}
async getMetadata(t) {
const e = await this.transaction("metadata");
return new Promise((s, n) => {
const a = e.get(t);
a.onsuccess = () => {
s(a.result);
}, a.onerror = (i) => n(i.target.error);
});
}
async getChunks(t) {
const e = await this.transaction("chunks");
return new Promise((s, n) => {
const i = e.index("fileId").getAll(t);
i.onsuccess = () => s(i.result), i.onerror = (d) => n(d.target.error);
});
}
async cleanupFileData(t) {
await this.init();
const e = this.db.transaction(["chunks", "metadata"], "readwrite"), n = e.objectStore("chunks").index("fileId"), a = IDBKeyRange.only(t);
return n.openCursor(a).onsuccess = (d) => {
const h = d.target.result;
h && (h.delete(), h.continue());
}, e.objectStore("metadata").delete(t), new Promise((d, h) => {
e.oncomplete = () => d(!0), e.onerror = () => h(e.error);
});
}
async cleanupExpiredChunks() {
const t = Date.now();
await this.init();
const e = this.db.transaction(["chunks", "metadata"], "readwrite"), s = IDBKeyRange.upperBound(t - U), i = e.objectStore("metadata").index("updateAt").openCursor(s);
await new Promise((c) => {
i.onsuccess = (g) => {
const m = g.target.result;
m && (m.delete(), m.continue()), c(!0);
};
});
const u = e.objectStore("chunks").index("updateAt");
return u.openCursor(s).onsuccess = (c) => {
const g = c.target.result;
g && (g.delete(), g.continue());
}, new Promise((c, g) => {
e.oncomplete = () => {
c(!0);
}, e.onerror = () => g(e.error);
});
}
close() {
this.db && (this.db.close(), this.db = null);
}
}
class C {
constructor(t = 1, e = "file_chunks_db") {
o(this, "_store");
o(this, "type");
this._store = k ? new $(t, e) : new T(), this.type = k ? "IndexedDB" : "Memory";
}
async checkChunk(t, e) {
return this._store.checkChunk(t, e);
}
async saveChunk(t, e, s, n) {
return this._store.saveChunk(t, e, s, n);
}
async updateMetadata(t, e) {
return this._store.updateMetadata(t, e);
}
async getMetadata(t) {
return this._store.getMetadata(t);
}
async getChunks(t) {
return this._store.getChunks(t);
}
async cleanupFileData(t) {
return this._store.cleanupFileData(t);
}
async cleanupExpiredChunks() {
return this._store.cleanupExpiredChunks();
}
close() {
this._store && this._store.close();
}
}
class G {
constructor(t) {
o(this, "options");
o(this, "event");
o(this, "storage");
o(this, "fileList");
this.options = S(B, t), this.event = new v(), this.storage = new C(this.options.storageVersion, this.options.storageName), this.fileList = [];
}
on(t, e) {
this.event.on(t, e);
}
emit(t, ...e) {
this.event.emit(t, ...e, this.fileList);
}
setOption(t) {
this.options = S(this.options, t), this.storage = new C(t.storageVersion, t.storageName);
}
async start(t) {
if (typeof t != "string" || !t.trim())
throw this.emit(p.FAILED, null, this.fileList), new Error("A valid URL is required");
if (!this.options.customRequest && !this.options.action)
throw new Error("Config action is required");
new _({ ...this.options, url: t }, this);
}
_addFile(t) {
this.fileList.some((e) => e.id === t.id) || this.fileList.push(t);
}
}
export {
p as Callbacks,
G as Downloader,
l as FileStatus
};