@lzwme/m3u8-dl
Version:
Batch download of m3u8 files and convert to mp4
324 lines (323 loc) • 14.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.workPollPublic = exports.downloadQueue = exports.DownloadQueue = void 0;
exports.preDownLoad = preDownLoad;
exports.m3u8Download = m3u8Download;
exports.m3u8DLStop = m3u8DLStop;
const node_fs_1 = require("node:fs");
const node_path_1 = require("node:path");
const fe_utils_1 = require("@lzwme/fe-utils");
const helper_1 = require("@lzwme/fe-utils/cjs/common/helper");
const console_log_colors_1 = require("console-log-colors");
const format_options_js_1 = require("./format-options.js");
const local_play_js_1 = require("./local-play.js");
const m3u8_convert_js_1 = require("./m3u8-convert.js");
const parseM3u8_js_1 = require("./parseM3u8.js");
const utils_js_1 = require("./utils.js");
const worker_pool_js_1 = require("./worker_pool.js");
/** 下载队列管理 */
class DownloadQueue {
queue = [];
activeDownloads = new Set();
_maxConcurrent;
constructor(maxConcurrent = 3) {
this._maxConcurrent = maxConcurrent;
}
get maxConcurrent() {
return this._maxConcurrent;
}
set maxConcurrent(value) {
this._maxConcurrent = value;
}
add(url, options, priority = 0) {
this.queue.push({ url, options, priority });
this.queue.sort((a, b) => b.priority - a.priority);
this.processQueue();
}
async processQueue() {
if (this.activeDownloads.size >= this._maxConcurrent)
return;
const next = this.queue.shift();
if (!next)
return;
this.activeDownloads.add(next.url);
try {
const { maxDownloads, ...options } = next.options;
const result = await m3u8Download(next.url, options);
next.options.onComplete?.(result);
}
catch (error) {
next.options.onComplete?.({
errmsg: error instanceof Error ? error.message : error ? JSON.stringify(error) : 'Unknown error',
options: next.options,
});
}
finally {
this.activeDownloads.delete(next.url);
this.processQueue();
}
}
pause(url) {
const index = this.queue.findIndex(item => item.url === url);
if (index > -1) {
this.queue.splice(index, 1);
}
}
resume(url, options, priority = 0) {
this.add(url, options, priority);
}
clear() {
this.queue = [];
}
getStatus() {
return {
queueLength: this.queue.length,
activeDownloads: Array.from(this.activeDownloads),
maxConcurrent: this._maxConcurrent,
};
}
}
exports.DownloadQueue = DownloadQueue;
/** 创建全局下载队列实例 */
exports.downloadQueue = new DownloadQueue();
const cache = {
m3u8Info: {},
downloading: new Set(),
};
const tsDlFile = (0, node_path_1.resolve)(__dirname, './ts-download.js');
exports.workPollPublic = new worker_pool_js_1.WorkerPool(tsDlFile);
async function m3u8InfoParse(u, o = {}) {
const { url, options, urlMd5 } = (0, format_options_js_1.formatOptions)(u, o);
const ext = (0, utils_js_1.isSupportFfmpeg)() ? '.mp4' : '.ts';
/** 最终合并转换后的文件路径 */
let filepath = (0, node_path_1.resolve)(options.saveDir, options.filename);
if (!filepath.endsWith(ext))
filepath += ext;
const result = { options, m3u8Info: null, filepath };
if (cache.m3u8Info[url]) {
Object.assign(result, cache.m3u8Info[url]);
return result;
}
if (!options.force && (0, node_fs_1.existsSync)(filepath))
return result;
const m3u8Info = await (0, parseM3u8_js_1.parseM3U8)(url, (0, node_path_1.resolve)(options.cacheDir, urlMd5), options.headers).catch(e => {
utils_js_1.logger.error('[parseM3U8][failed]', e.message);
console.log(e);
});
if (m3u8Info && m3u8Info?.tsCount > 0) {
result.m3u8Info = m3u8Info;
if (options.ignoreSegments) {
const timeSegments = options.ignoreSegments
.split(',')
.map(d => d.split(/[- ]+/).map(d => +d.trim()))
.filter(d => d[0] && d[1] && d[0] !== d[1]);
if (timeSegments.length) {
const total = m3u8Info.data.length;
m3u8Info.data = m3u8Info.data.filter(item => {
for (let [start, end] of timeSegments) {
if (start > end)
[start, end] = [end, start];
if (item.timeline + item.duration / 2 >= start && item.timeline + item.duration / 2 <= end) {
m3u8Info.duration -= item.duration;
return false;
}
}
return true;
});
const ignoredCount = total - m3u8Info.data.length;
if (ignoredCount) {
m3u8Info.tsCount = m3u8Info.data.length;
utils_js_1.logger.info(`[parseM3U8][ignoreSegments] ignored ${(0, console_log_colors_1.cyanBright)(ignoredCount)} segments`);
m3u8Info.duration = +Number(m3u8Info.duration).toFixed(2);
}
}
}
}
return result;
}
/**
* 计算实时下载速度：取从 idx 开始的最近 maxTime 时间内的下载总大小
*/
function calcSpeed(idx, data, maxTime = 60 * 1000) {
const now = Date.now();
let startTime = now;
let downloadedSize = 0;
for (let i = idx; i >= 0; i--) {
const info = data[i];
if (info.tsSize && info.startTime && now - info.startTime < maxTime) {
if (startTime > info.startTime)
startTime = info.startTime;
downloadedSize += info.tsSize;
}
}
return now > startTime ? 1000 * (downloadedSize / (now - startTime)) : 0;
}
async function preDownLoad(url, options, wp = exports.workPollPublic) {
const result = await m3u8InfoParse(url, options);
if (!result.m3u8Info)
return;
for (const info of result.m3u8Info.data) {
if (!wp.freeNum)
return;
if (!cache.downloading.has(info.uri)) {
cache.downloading.add(info.uri);
wp.runTask({ url, info, options: JSON.parse(JSON.stringify(result.options)), crypto: result.m3u8Info.crypto[info.keyUri] }, () => {
cache.downloading.delete(info.uri);
});
}
}
}
async function m3u8Download(url, options = {}) {
// 如果设置了最大并发数，则使用队列管理
if (options.maxDownloads) {
exports.downloadQueue.maxConcurrent = options.maxDownloads;
return new Promise(resolve => {
const newOptions = {
...options,
onComplete: r => {
if (options.onComplete)
options.onComplete(r);
resolve(r);
},
};
exports.downloadQueue.add(url, newOptions, options.priority || 0);
});
}
// 原有的下载逻辑
utils_js_1.logger.info('Starting download for', (0, console_log_colors_1.cyanBright)(url));
const result = await m3u8InfoParse(url, options);
options = result.options;
if (!options.force && (0, node_fs_1.existsSync)(result.filepath) && !result.m3u8Info) {
utils_js_1.logger.info('file already exist:', result.filepath);
result.isExist = true;
return result;
}
if (result.m3u8Info?.tsCount > 0) {
const workPoll = new worker_pool_js_1.WorkerPool(tsDlFile, options.threadNum);
const { m3u8Info } = result;
const startTime = Date.now();
const barrier = new fe_utils_1.Barrier();
/** 本地开始播放最少需要下载的 ts 文件数量 */
const playStart = Math.min(options.threadNum + 2, m3u8Info.tsCount);
const stats = {
url,
startTime,
progress: 0,
tsCount: m3u8Info.tsCount,
tsSuccess: 0,
tsFailed: 0,
duration: m3u8Info.duration,
durationDownloaded: 0,
downloadedSize: 0,
avgSpeed: 0,
avgSpeedDesc: '',
speed: 0,
speedDesc: '',
remainingTime: 0,
localM3u8: (0, local_play_js_1.toLocalM3u8)(m3u8Info.data).replace(options.cacheDir, '').replaceAll(node_path_1.sep, '/').slice(1),
filename: options.filename,
threadNum: options.threadNum,
};
const runTask = (data) => {
for (const info of data) {
const o = JSON.parse(JSON.stringify(options));
workPoll.runTask({ url, info, options: o, crypto: m3u8Info.crypto[info.keyUri] }, (err, res, taskStartTime) => {
stats.errmsg = err ? err.cause || err.message || err.toString() : '';
if (!res || err) {
if (err) {
console.log('\n');
utils_js_1.logger.error('[TS-DL][error]', info.index, err, res || '');
}
if (typeof info.success !== 'number')
info.success = 0;
else
info.success--;
if (info.success >= -3) {
utils_js_1.logger.warn(`[retry][times: ${info.success}]`, info.index, info.uri);
setTimeout(() => runTask([info]), 1000);
return;
}
}
if (res?.success) {
info.tsSize = res.info.tsSize;
info.startTime = taskStartTime; // res.timeCost;
info.timeCost = Date.now() - taskStartTime;
info.success = 1;
stats.tsSuccess++;
stats.durationDownloaded += info.duration;
stats.speed = calcSpeed(info.index, data);
}
else {
stats.tsFailed++;
}
const finished = stats.tsFailed + stats.tsSuccess;
const timeCost = Date.now() - startTime;
const downloadedDuration = m3u8Info.data.reduce((a, b) => a + (b.tsSize ? b.duration : 0), 0);
stats.downloadedSize = m3u8Info.data.reduce((a, b) => a + (b.tsSize || 0), 0);
stats.avgSpeed = (stats.downloadedSize / timeCost) * 1000;
stats.avgSpeedDesc = `${(0, helper_1.formatByteSize)(stats.avgSpeed)}/s`;
// 如果当前速度小于平均速度，则更新为平均速度
if (stats.speed < stats.avgSpeed)
stats.speed = stats.avgSpeed;
stats.speedDesc = `${(0, helper_1.formatByteSize)(stats.speed)}/s`;
stats.progress = Math.floor((finished / m3u8Info.tsCount) * 100);
if (downloadedDuration) {
stats.remainingTime = (timeCost / downloadedDuration) * (m3u8Info.duration - stats.durationDownloaded);
if (stats.speed > stats.avgSpeed)
stats.remainingTime = stats.remainingTime * (stats.avgSpeed / stats.speed);
stats.remainingTime = Math.ceil(stats.remainingTime);
stats.size = Math.round(stats.downloadedSize * (stats.duration / stats.durationDownloaded));
}
if (options.showProgress) {
const processBar = '='.repeat(Math.floor(stats.progress * 0.2)).padEnd(20, '-');
utils_js_1.logger.logInline(`${stats.progress}% [${(0, console_log_colors_1.greenBright)(processBar)}] ${(0, console_log_colors_1.cyan)(finished)} ${(0, console_log_colors_1.green)(`${stats.durationDownloaded.toFixed(2)}sec`)} ${(0, console_log_colors_1.blueBright)((0, helper_1.formatByteSize)(stats.downloadedSize))} ${(0, console_log_colors_1.yellowBright)((0, fe_utils_1.formatTimeCost)(startTime))} ${(0, console_log_colors_1.magentaBright)(stats.speedDesc)} ${finished === m3u8Info.tsCount
? '\n'
: stats.remainingTime
? `${(0, console_log_colors_1.cyan)((0, fe_utils_1.formatTimeCost)(Date.now() - stats.remainingTime))}`
: ''}`);
}
if (options.onProgress)
options.onProgress(finished, m3u8Info.tsCount, info, stats);
if (finished === m3u8Info.tsCount) {
// pool.close();
barrier.open();
}
if (options.play && finished === playStart)
(0, local_play_js_1.localPlay)(m3u8Info.data);
});
}
};
if (options.showProgress) {
console.info(`\nTotal segments: ${(0, console_log_colors_1.cyan)(m3u8Info.tsCount)}, duration: ${(0, console_log_colors_1.green)(`${m3u8Info.duration}sec`)}.`, `Parallel jobs: ${(0, console_log_colors_1.magenta)(options.threadNum)}`);
}
result.stats = stats;
(0, local_play_js_1.toLocalM3u8)(m3u8Info.data);
if (options.onInited)
options.onInited(stats, m3u8Info, workPoll);
runTask(m3u8Info.data);
await barrier.wait();
workPoll.close();
if (stats.tsFailed > 0) {
utils_js_1.logger.warn('Download Failed! Please retry!', stats.tsFailed);
}
else if (options.convert !== false) {
result.filepath = await (0, m3u8_convert_js_1.m3u8Convert)(options, m3u8Info.data);
if (result.filepath && (0, node_fs_1.existsSync)(result.filepath)) {
stats.size = (0, node_fs_1.statSync)(result.filepath).size;
if (options.delCache)
(0, fe_utils_1.rmrfAsync)((0, node_path_1.dirname)(m3u8Info.data[0].tsOut));
}
}
}
utils_js_1.logger.debug('Done!', url, result.m3u8Info);
return result;
}
function m3u8DLStop(url, wp = exports.workPollPublic) {
if (!wp?.removeTask)
return 0;
const count = wp.removeTask(task => task.url === url);
// 进行中的任务，最多允许继续下载 10s
if (count === 0 && wp !== exports.workPollPublic)
setTimeout(() => wp.close(), 10_000);
return count;
}