@lzwme/m3u8-dl
Version:
Batch download of m3u8 files and convert to mp4
580 lines (579 loc) • 26.7 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.DLServer = void 0;
const node_fs_1 = require("node:fs");
const node_path_1 = require("node:path");
const node_os_1 = require("node:os");
const fe_utils_1 = require("@lzwme/fe-utils");
const console_log_colors_1 = require("console-log-colors");
const file_download_js_1 = require("../lib/file-download.js");
const format_options_js_1 = require("../lib/format-options.js");
const m3u8_download_js_1 = require("../lib/m3u8-download.js");
const getM3u8Urls_js_1 = require("../lib/getM3u8Urls.js");
const utils_js_1 = require("../lib/utils.js");
const index_js_1 = require("../video-parser/index.js");
const rootDir = (0, node_path_1.resolve)(__dirname, '../..');
class DLServer {
app = null;
wss = null;
/** DS 参数 */
options = {
port: Number(process.env.DS_PORT) || 6600,
cacheDir: process.env.DS_CACHE_DIR || (0, node_path_1.resolve)((0, node_os_1.homedir)(), '.m3u8-dl/cache'),
token: process.env.DS_SECRET || process.env.DS_TOKEN || '',
debug: process.env.DS_DEBUG === '1',
limitFileAccess: ['1', 'true'].includes(process.env.DS_LIMTE_FILE_ACCESS),
};
serverInfo = {
version: '',
ariang: (0, node_fs_1.existsSync)((0, node_path_1.resolve)(rootDir, 'client/ariang/index.html')),
};
cfg = {
/** 支持 web 设置修改的参数 */
webOptions: {
/** 最多同时下载数量。超过改值则设置为 pending */
maxDownloads: 3,
/** 是否显示预览按钮 */
showPreview: true,
/** 是否显示边下边播按钮 */
showLocalPlay: true,
},
/** download 下载默认参数 */
dlOptions: {
debug: process.env.DS_DEBUG === '1',
saveDir: process.env.DS_SAVE_DIR || './downloads',
threadNum: 4,
},
};
/** 下载任务缓存 */
dlCache = new Map();
/** 正在下载的任务数量 */
get downloading() {
return Array.from(this.dlCache.values()).filter(item => item.status === 'resume').length;
}
constructor(opts = {}) {
opts = Object.assign(this.options, opts);
opts.cacheDir = (0, node_path_1.resolve)(opts.cacheDir);
if (!opts.configPath)
opts.configPath = (0, node_path_1.resolve)(opts.cacheDir, 'config.json');
const pkgFile = (0, node_path_1.resolve)(rootDir, 'package.json');
if ((0, node_fs_1.existsSync)(pkgFile)) {
const pkg = JSON.parse((0, node_fs_1.readFileSync)(pkgFile, 'utf8'));
this.serverInfo.version = pkg.version;
}
if (opts.token)
opts.token = (0, fe_utils_1.md5)(opts.token.trim()).slice(0, 8);
this.init();
}
async init() {
this.readConfig();
if (this.cfg.dlOptions.debug)
utils_js_1.logger.updateOptions({ levelType: 'debug' });
this.loadCache();
await this.createApp();
this.initRouters();
utils_js_1.logger.debug('Server initialized', 'cacheSize:', this.dlCache.size, this.options, this.cfg.dlOptions);
}
loadCache() {
const cacheFile = (0, node_path_1.resolve)(this.options.cacheDir, 'cache.json');
if ((0, node_fs_1.existsSync)(cacheFile)) {
JSON.parse((0, node_fs_1.readFileSync)(cacheFile, 'utf8')).forEach(([url, item]) => {
if (item.status === 'resume')
item.status = 'pause';
this.dlCache.set(url, item);
});
this.checkDLFileIsExists();
}
}
checkDLFileLaest = 0;
checkDLFileTimer = null;
checkDLFileIsExists() {
const now = Date.now();
const interval = 1000 * 60;
clearTimeout(this.checkDLFileTimer);
if (now - this.checkDLFileLaest < interval) {
this.checkDLFileTimer = setTimeout(() => this.checkDLFileIsExists(), interval - (now - this.checkDLFileLaest));
return;
}
this.dlCache.forEach(item => {
if (item.status === 'done' && (!item.localVideo || !(0, node_fs_1.existsSync)(item.localVideo))) {
item.status = 'error';
item.errmsg = '已删除';
}
});
}
dlCacheClone() {
const info = [];
for (const [url, v] of this.dlCache) {
const { workPoll, ...item } = v;
for (const [key, value] of Object.entries(item)) {
if (typeof value === 'function')
delete item[key];
}
info.push([url, item]);
}
return info;
}
cacheSaveTimer = null;
saveCache() {
clearTimeout(this.cacheSaveTimer);
this.cacheSaveTimer = setTimeout(() => {
const cacheFile = (0, node_path_1.resolve)(this.options.cacheDir, 'cache.json');
const info = this.dlCacheClone();
(0, fe_utils_1.mkdirp)((0, node_path_1.dirname)(cacheFile));
(0, node_fs_1.writeFileSync)(cacheFile, JSON.stringify(info));
}, 1000);
}
readConfig(configPath) {
try {
if (!configPath)
configPath = this.options.configPath;
if ((0, node_fs_1.existsSync)(configPath))
(0, fe_utils_1.assign)(this.cfg, JSON.parse((0, node_fs_1.readFileSync)(configPath, 'utf8')));
}
catch (error) {
utils_js_1.logger.error('读取配置失败:', error);
}
return this.cfg.dlOptions;
}
saveConfig(config, configPath) {
if (!configPath)
configPath = this.options.configPath;
for (const [key, value] of Object.entries(config)) {
// @ts-expect-error 忽略类型错误
if (key in this.cfg.webOptions)
this.cfg.webOptions[key] = value;
// @ts-expect-error 忽略类型错误
else
this.cfg.dlOptions[key] = value;
}
(0, fe_utils_1.mkdirp)((0, node_path_1.dirname)(configPath));
(0, node_fs_1.writeFileSync)(configPath, JSON.stringify(this.cfg, null, 2));
}
async createApp() {
const { default: express } = await Promise.resolve().then(() => __importStar(require('express')));
const { WebSocketServer } = await Promise.resolve().then(() => __importStar(require('ws')));
const app = express();
const server = app.listen(this.options.port, () => utils_js_1.logger.info(`Server running on port ${(0, console_log_colors_1.green)(this.options.port)}`));
const wss = new WebSocketServer({ server });
this.app = app;
this.wss = wss;
app.use((req, res, next) => {
if (['/', '/index.html'].includes(req.path)) {
const version = this.serverInfo.version;
let indexHtml = (0, node_fs_1.readFileSync)((0, node_path_1.resolve)(rootDir, 'client/index.html'), 'utf-8').replaceAll('{{version}}', version);
if ((0, node_fs_1.existsSync)((0, node_path_1.resolve)(rootDir, 'client/local/cdn'))) {
indexHtml = indexHtml
.replaceAll('https://s4.zstatic.net/ajax/libs', 'local/cdn')
.replaceAll(/integrity="[^"]+"\n?/g, '')
.replace('https://cdn.tailwindcss.com/3.4.16', 'local/cdn/tailwindcss/3.4.16/tailwindcss.min.js');
}
res.setHeader('content-type', 'text/html').send(indexHtml);
}
else {
next();
}
});
app.use(express.json());
app.use(express.static((0, node_path_1.resolve)(rootDir, 'client')));
app.use((req, res, next) => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
res.setHeader('Cache-Control', 'no-cache');
if (req.method === 'OPTIONS') {
res.status(200).end();
return;
}
if (this.options.token && req.headers.authorization !== this.options.token) {
const ignorePaths = ['/healthcheck', '/localplay'];
if (!ignorePaths.some(d => req.url.includes(d))) {
const clientIp = req.headers['x-forwarded-for'] || req.socket.remoteAddress;
utils_js_1.logger.warn('Unauthorized access:', clientIp, req.url, req.headers.authorization);
res.status(401).json({ message: '未授权，禁止访问', code: 401 });
return;
}
}
next();
});
wss.on('connection', (ws, req) => {
const clientIp = req.headers['x-forwarded-for'] || req.socket.remoteAddress;
utils_js_1.logger.info('Client connected:', clientIp, req.url);
if (this.options.token) {
const token = (0, fe_utils_1.getUrlParams)(req.url).token;
if (!token || token !== this.options.token) {
utils_js_1.logger.error('Unauthorized client:', req.socket.remoteAddress);
ws.close(1008, 'Unauthorized');
return;
}
}
this.checkDLFileIsExists();
ws.send(JSON.stringify({ type: 'serverInfo', data: this.serverInfo }));
ws.send(JSON.stringify({ type: 'tasks', data: Object.fromEntries(this.dlCacheClone()) }));
});
wss.on('close', () => utils_js_1.logger.info('WebSocket server closed'));
wss.on('error', err => utils_js_1.logger.error('WebSocket server error:', err));
wss.on('listening', () => utils_js_1.logger.info(`WebSocket server listening on port ${(0, console_log_colors_1.green)(this.options.port)}`));
return { app, wss };
}
async startDownload(url, options) {
if (!url)
return utils_js_1.logger.error('[satartDownload]Invalid URL:', url);
if (url.endsWith('.html')) {
const item = Array.from(await (0, getM3u8Urls_js_1.getM3u8Urls)({ url, headers: options.headers }))[0];
if (!item)
return utils_js_1.logger.error('[startDownload]不是有效(包含)M3U8的地址:', url);
url = item[0];
if (!options.filename)
options.filename = item[1];
}
const { options: dlOptions } = (0, format_options_js_1.formatOptions)(url, { ...this.cfg.dlOptions, ...options, cacheDir: this.options.cacheDir });
const cacheItem = this.dlCache.get(url) || { options, dlOptions, status: 'pending', url };
utils_js_1.logger.debug('startDownload', url, dlOptions, cacheItem.status);
if (cacheItem.status === 'resume')
return;
if (cacheItem.localVideo && !(0, node_fs_1.existsSync)(cacheItem.localVideo))
delete cacheItem.localVideo;
if (cacheItem.endTime)
delete cacheItem.endTime;
cacheItem.status = this.downloading >= this.cfg.webOptions.maxDownloads ? 'pending' : 'resume';
// pending 优先级靠后
if (cacheItem.status === 'pending' && this.dlCache.has(url))
this.dlCache.delete(url);
this.dlCache.set(url, cacheItem);
this.wsSend('progress', url);
if (cacheItem.status === 'pending')
return;
let workPoll = cacheItem.workPoll;
const opts = {
...dlOptions,
showProgress: dlOptions.debug || this.options.debug,
onInited: (_s, _i, wp) => {
workPoll = wp;
},
onProgress: (_finished, _total, current, stats) => {
const item = this.dlCache.get(url);
if (!item)
return false; // 已删除
const status = item.status || 'resume';
Object.assign(item, { ...stats, current, options: dlOptions, status, workPoll, url });
this.dlCache.set(url, item);
this.saveCache();
this.wsSend('progress', url);
return status !== 'pause';
},
};
const afterDownload = (r, url) => {
const item = this.dlCache.get(url) || cacheItem;
if (r.filepath && (0, node_fs_1.existsSync)(r.filepath)) {
item.localVideo = r.filepath;
item.downloadedSize = (0, node_fs_1.statSync)(r.filepath).size;
}
else if (!r.errmsg && opts.convert !== false)
r.errmsg = '下载失败';
item.endTime = Date.now();
item.errmsg = r.errmsg;
item.status = r.errmsg ? 'error' : 'done';
utils_js_1.logger.info('Download complete:', item.status, (0, console_log_colors_1.red)(r.errmsg), (0, console_log_colors_1.gray)(url), (0, console_log_colors_1.cyan)(r.filepath));
this.dlCache.set(url, item);
this.wsSend('progress', url);
this.saveCache();
this.startNextPending();
};
try {
if (dlOptions.type === 'parser') {
const vp = new index_js_1.VideoParser();
vp.download(url, opts).then(r => afterDownload(r, url));
}
else if (dlOptions.type === 'file') {
(0, file_download_js_1.fileDownload)(url, opts).then(r => afterDownload(r, url));
}
else {
(0, m3u8_download_js_1.m3u8Download)(url, opts).then(r => afterDownload(r, url));
}
}
catch (error) {
afterDownload({ filepath: '', errmsg: error.message }, url);
utils_js_1.logger.error('下载失败:', error);
}
}
startNextPending() {
// 找到一个 pending 的任务，开始下载
const nextItem = this.dlCache.entries().find(([_url, d]) => d.status === 'pending');
if (nextItem) {
this.startDownload(nextItem[0], nextItem[1].options);
this.wsSend('progress', nextItem[0]);
}
}
wsSend(type = 'progress', data) {
if (type === 'tasks' && !data) {
data = Object.fromEntries(this.dlCacheClone());
}
else if (type === 'progress' && typeof data === 'string') {
const item = this.dlCache.get(data);
if (!item)
return;
const { workPoll, ...stats } = item;
data = [{ ...stats, url: data }];
}
// 广播进度信息给所有客户端
this.wss.clients.forEach(client => {
if (client.readyState === 1)
client.send(JSON.stringify({ type, data }));
});
}
initRouters() {
const { app } = this;
app.get('/healthcheck', (_req, res) => {
res.json({ message: 'ok', code: 0 });
});
app.post('/api/config', (req, res) => {
const config = req.body;
this.saveConfig(config);
res.json({ message: 'Config updated successfully', code: 0 });
});
app.get('/api/config', (_req, res) => {
res.json({ ...this.cfg.dlOptions, ...this.cfg.webOptions });
});
// API to get all download progress
app.get('/api/tasks', (_req, res) => {
res.json(Object.fromEntries(this.dlCacheClone()));
});
// API to get queue status
app.get('/api/queue/status', (_req, res) => {
const pendingTasks = Array.from(this.dlCache.entries()).filter(([_, item]) => item.status === 'pending');
const activeTasks = Array.from(this.dlCache.entries()).filter(([_, item]) => item.status === 'resume');
res.json({
queueLength: pendingTasks.length,
activeDownloads: activeTasks.map(([url]) => url),
maxConcurrent: this.cfg.webOptions.maxDownloads,
});
});
// API to clear queue
app.post('/api/queue/clear', (_req, res) => {
let count = 0;
for (const [url, item] of this.dlCache.entries()) {
if (item.status === 'pending') {
this.dlCache.delete(url);
count++;
}
}
if (count)
this.wsSend('tasks');
res.json({ message: `已清空 ${count} 个等待中的下载任务`, code: 0 });
});
// API to update task priority
app.post('/api/priority', (req, res) => {
const { url, priority } = req.body;
const item = this.dlCache.get(url);
if (!item) {
res.json({ message: '任务不存在', code: 1 });
return;
}
item.options.priority = priority;
this.saveCache();
res.json({ message: '已更新任务优先级', code: 0 });
});
// API to start m3u8 download
app.post('/api/download', (req, res) => {
const { url, options = {}, list = [] } = req.body;
try {
if (list.length) {
for (const item of list) {
const { url, ...options } = item;
if (url)
this.startDownload(url, options);
}
}
else if (url)
this.startDownload(url, options);
res.json({ message: `Download started: ${list.length || 1}`, code: 0 });
this.wsSend('tasks');
}
catch (error) {
res.status(500).json({ error: `Download failed: ${error.message}` });
}
});
// API to pause download
app.post('/api/pause', (req, res) => {
const { urls, all = false } = req.body;
const urlsToPause = all ? [...this.dlCache.keys()] : urls;
const list = [];
for (const url of urlsToPause) {
const item = this.dlCache.get(url);
if (['resume', 'pending'].includes(item?.status)) {
(0, m3u8_download_js_1.m3u8DLStop)(url, item.workPoll);
item.status = item.tsSuccess > 0 && item.tsSuccess === item.tsCount ? 'done' : 'pause';
const { workPoll, ...tItem } = item;
list.push(tItem);
}
}
if (list.length) {
this.wsSend('progress', list);
this.startNextPending();
}
res.json({ message: `已暂停 ${list.length} 个下载任务`, code: 0, count: list.length });
});
// API to resume download
app.post('/api/resume', (req, res) => {
const { urls, all = false } = req.body;
const urlsToResume = all ? [...this.dlCache.keys()] : urls;
const list = [];
for (const url of urlsToResume) {
const item = this.dlCache.get(url);
if (['pause', 'error'].includes(item?.status)) {
this.startDownload(url, item.options);
const { workPoll, ...t } = item;
list.push(t);
}
else
console.log(item?.status, url);
}
if (list.length)
this.wsSend('progress', list);
res.json({ message: list.length ? `已恢复 ${list.length} 个下载任务` : '没有找到可恢复的下载任务', code: 0, count: list.length });
});
// API to delete download
app.post('/api/delete', (req, res) => {
const { urls, deleteCache = false, deleteVideo = false } = req.body;
const urlsToDelete = urls;
const list = [];
for (const url of urlsToDelete) {
const item = this.dlCache.get(url);
if (item) {
(0, m3u8_download_js_1.m3u8DLStop)(url, item.workPoll);
this.dlCache.delete(url);
list.push(item.url);
if (deleteCache && item.current?.tsOut) {
const cacheDir = (0, node_path_1.dirname)(item.current.tsOut);
if ((0, node_fs_1.existsSync)(cacheDir)) {
(0, node_fs_1.rmSync)(cacheDir, { recursive: true });
utils_js_1.logger.debug('删除缓存目录：', cacheDir);
}
}
if (deleteVideo) {
['.ts', '.mp4'].forEach(ext => {
const filepath = (0, node_path_1.resolve)(item.options.saveDir, item.options.filename + ext);
if ((0, node_fs_1.existsSync)(filepath)) {
(0, node_fs_1.unlinkSync)(filepath);
utils_js_1.logger.debug('删除文件：', filepath);
}
});
}
}
}
if (list.length) {
this.wsSend('delete', list);
this.saveCache();
this.startNextPending();
}
res.json({ message: `已删除 ${list.length} 个下载任务`, code: 0, count: list.length });
});
app.get(/^\/localplay\/(.*)$/, (req, res) => {
let filepath = decodeURIComponent(req.params[0]);
if (filepath) {
let ext = filepath.split('.').pop();
if (!ext) {
ext = 'm3u8';
if (!(0, node_fs_1.existsSync)(filepath))
filepath += '.m3u8';
}
const allowedDirs = [this.options.cacheDir, this.cfg.dlOptions.saveDir];
if (!(0, node_fs_1.existsSync)(filepath)) {
for (const dir of allowedDirs) {
const tpath = (0, node_path_1.resolve)(dir, filepath);
if ((0, node_fs_1.existsSync)(tpath)) {
filepath = tpath;
break;
}
}
}
else {
filepath = (0, node_path_1.resolve)(filepath);
const isAllow = !this.options.limitFileAccess || allowedDirs.some(d => filepath.startsWith((0, node_path_1.resolve)(d)));
if (!isAllow) {
utils_js_1.logger.error('[Localplay] Access denied:', filepath);
res.send({ message: 'Access denied', code: 403 });
return;
}
}
if ((0, node_fs_1.existsSync)(filepath)) {
const stats = (0, node_fs_1.statSync)(filepath);
const headers = new Headers({
'Last-Modified': stats.mtime.toUTCString(),
'Access-Control-Allow-Headers': '*',
'Access-Control-Allow-Methods': 'POST, GET, OPTIONS',
'Access-Control-Allow-Origin': '*',
'Content-Length': String(stats.size),
'Content-Type': ext === 'ts' ? 'video/mp2t' : ext === 'm3u8' ? 'application/vnd.apple.mpegurl' : ext === 'mp4' ? 'video/mp4' : 'text/plain',
});
res.setHeaders(headers);
if (ext === 'm3u8' || ('ts' === ext && stats.size < 1024 * 1024 * 3)) {
let content = (0, node_fs_1.readFileSync)(filepath);
if (ext === 'm3u8') {
const baseDirName = (0, node_path_1.basename)(filepath, '.m3u8');
content = content
.toString('utf8')
.split('\n')
.map(line => (line.endsWith('.ts') && !line.includes('/') ? `${baseDirName}/${line}` : line))
.join('\n');
}
res.send(content);
utils_js_1.logger.debug('[Localplay]file sent:', (0, console_log_colors_1.gray)(filepath), 'Size:', stats.size, 'bytes');
}
else {
res.sendFile(filepath);
}
return;
}
}
utils_js_1.logger.error('[Localplay]file not found:', (0, console_log_colors_1.red)(filepath));
res.status(404).send({ message: 'Not Found', code: 404 });
});
app.post('/api/getM3u8Urls', (req, res) => {
const { url, headers, subUrlRegex } = req.body;
if (!url) {
res.json({ code: 1001, message: '无效的 url 参数' });
}
else {
(0, getM3u8Urls_js_1.getM3u8Urls)({ url, headers, subUrlRegex })
.then(d => res.json({ code: 0, data: Array.from(d) }))
.catch(err => res.json({ code: 401, message: err.message }));
}
});
}
}
exports.DLServer = DLServer;