mira-app-server
Version:
Mira Server - standalone server application using mira-app-core
216 lines • 9.79 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadExecutorService = void 0;
const axios_1 = __importDefault(require("axios"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const promises_1 = require("stream/promises");
const crypto_1 = require("crypto");
const p_queue_1 = __importDefault(require("p-queue"));
/**
* 下载执行器：消费用户已配置的 cookie 站点，把一批图片 URL 下载并入库到指定素材库。
*
* 流程（每个 URL）：
* 1. 按 host 匹配该用户的 cookie 组（is_default 优先，否则同 url 第一组），拼 Cookie header
* 2. axios stream 下载到 backend.dataPath/temp
* 3. libraryService.createFileFromPath(tmp, { uploader, folder_id }, { importType:'move' })
* —— 自动 hash 去重 + 文件搬运 + 触发 file::created（生成缩略图）
* 4. WebSocket 推送 download::progress / download::item 到对应 library
*
* 并发用 p-queue（同 MetadataService 模式）；进度存内存 Map，重启清空。
*/
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
class DownloadExecutorService {
constructor(backend) {
this.queue = new p_queue_1.default({ concurrency: 3 });
this.progress = new Map();
this.backend = backend;
}
/** 拿 UserStorage（通过 httpServer.authRouter） */
getUserStorage() {
return this.backend.httpServer.authRouter.getAuthService().getUserStorage();
}
/** 入队一批下载任务，返回 batchId */
async enqueueBatch(tasks) {
const batchId = (0, crypto_1.randomUUID)();
const progress = {
batchId,
total: tasks.length,
completed: 0,
failed: 0,
skipped: 0,
done: false,
};
this.progress.set(batchId, progress);
console.log(`📥 [download] batch ${batchId} 入队，共 ${tasks.length} 个任务`);
for (const task of tasks) {
void this.queue.add(() => this.runOne(batchId, task, progress));
}
return batchId;
}
getProgress(batchId) {
return this.progress.get(batchId) ?? null;
}
async runOne(batchId, task, progress) {
const { url, libraryId, userId, folderId, clientId } = task;
const ws = this.backend.webSocketServer;
const t0 = Date.now();
try {
// 校验 library
const libObj = this.backend.libraries?.getLibrary(libraryId);
if (!libObj?.libraryService)
throw new Error('素材库不可用');
// 1. 匹配 cookie
const cookieHeader = await this.matchCookieHeader(userId, url);
// 2. 下载到 temp
const tempDir = path.join(this.backend.dataPath, 'temp');
if (!fs.existsSync(tempDir))
fs.mkdirSync(tempDir, { recursive: true });
const basename = this.guessFilename(url);
const tmpPath = path.join(tempDir, `${Date.now()}-${Math.random().toString(36).slice(2, 8)}-${basename}`);
console.log(`⬇️ [download] batch ${batchId} 开始: ${url}${cookieHeader ? ' 【携带cookie】' : ''}`);
const resp = await axios_1.default.get(url, {
responseType: 'stream',
headers: {
'User-Agent': USER_AGENT,
...(cookieHeader ? { Cookie: cookieHeader } : {}),
},
timeout: 60000,
maxRedirects: 5,
});
await (0, promises_1.pipeline)(resp.data, fs.createWriteStream(tmpPath));
// 3. 入库（move 自动清理 temp + hash 去重）
const result = await libObj.libraryService.createFileFromPath(tmpPath, { uploader: userId, folder_id: folderId ?? null }, { importType: 'move' });
const isDup = result?.duplicate === true;
console.log(`✅ [download] batch ${batchId} ${isDup ? '重复跳过' : '完成'} (${Date.now() - t0}ms): ${url} → fileId=${result?.id}`);
if (!isDup && result?.id != null) {
// 补写来源 URL 到 custom_fields，便于追溯（与插件窗口 addFromUrl 模式一致）
try {
await libObj.libraryService.updateFile(result.id, {
custom_fields: { ...(result.custom_fields || {}), source_url: url },
});
}
catch { /* 元数据写入失败不影响下载结果 */ }
const eventData = {
...result,
libraryId,
batchImport: true,
batchId,
};
ws?.broadcastPluginEvent('file::created', {
message: { type: 'file', action: 'create' },
result: eventData,
libraryId,
});
ws?.broadcastLibraryEvent(libraryId, 'file::created', eventData);
}
if (isDup)
progress.skipped++;
else
progress.completed++;
// 4. 推送
ws?.broadcastLibraryEvent(libraryId, 'download::item', {
batchId, url, status: isDup ? 'duplicate' : 'success', file: result, libraryId,
});
if (clientId) {
const wsClient = ws?.getWsClientById(libraryId, clientId);
wsClient && ws?.sendToWebsocket(wsClient, {
eventName: 'download::item',
data: { batchId, url, status: isDup ? 'duplicate' : 'success', libraryId },
});
}
}
catch (e) {
progress.failed++;
console.log(`❌ [download] batch ${batchId} 失败 (${Date.now() - t0}ms): ${url} → ${e?.message || e}`);
ws?.broadcastLibraryEvent(libraryId, 'download::item', {
batchId, url, status: 'failed', error: e?.message || String(e), libraryId,
});
}
finally {
// 更新整体进度并推送
progress.done = progress.completed + progress.failed + progress.skipped >= progress.total;
if (progress.done) {
console.log(`📦 [download] batch ${batchId} 全部完成: 成功 ${progress.completed} / 失败 ${progress.failed} / 重复 ${progress.skipped}`);
}
ws?.broadcastLibraryEvent(libraryId, 'download::progress', { ...progress, libraryId });
if (clientId) {
const wsClient = ws?.getWsClientById(libraryId, clientId);
wsClient && ws?.sendToWebsocket(wsClient, {
eventName: 'download::progress',
data: { ...progress, libraryId },
});
}
}
}
/** 按 host 匹配该用户的 cookie 组：is_default 优先，否则该 url 第一组。返回 "a=b; c=d" 或空串 */
async matchCookieHeader(userId, url) {
try {
const host = new URL(url).host;
const sites = await this.getUserStorage().listCookieSites(userId);
const sameHost = sites.filter((s) => {
try {
return new URL(s.url).host === host;
}
catch {
return false;
}
});
const chosen = sameHost.find((s) => s.isDefault) || sameHost[0];
const cookies = chosen?.cookies || [];
return cookies.map((c) => `${c.name}=${c.value}`).filter(Boolean).join('; ');
}
catch {
return '';
}
}
/** 从 URL 猜测文件名，失败则用占位名 */
guessFilename(url) {
try {
const u = new URL(url);
const last = u.pathname.split('/').filter(Boolean).pop();
if (last && /\.[A-Za-z0-9]{2,5}$/.test(last))
return last.slice(0, 120);
}
catch { /* ignore */ }
return `mira-${Date.now()}.bin`;
}
}
exports.DownloadExecutorService = DownloadExecutorService;
//# sourceMappingURL=DownloadExecutorService.js.map