twspace-crawler
Version:
Script to monitor & download Twitter Spaces 24/7
196 lines • 8.54 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SpaceDownloader = void 0;
const axios_1 = __importDefault(require("axios"));
const bottleneck_1 = __importDefault(require("bottleneck"));
const child_process_1 = require("child_process");
const crypto_1 = require("crypto");
const fs_1 = require("fs");
const path_1 = __importDefault(require("path"));
const PeriscopeApi_1 = require("../apis/PeriscopeApi");
const logger_1 = require("../logger");
const m3u8_util_1 = require("../utils/m3u8.util");
const PeriscopeUtil_1 = require("../utils/PeriscopeUtil");
const Util_1 = require("../utils/Util");
const ConfigManager_1 = require("./ConfigManager");
/**
* Download audio OR video
*
* @see https://github.com/yt-dlp/yt-dlp/pull/10789
* @see https://github.com/HitomaruKonpaku/twspace-crawler/issues/93
*
*/
class SpaceDownloader {
constructor(originUrl, filename, subDir = '', metadata, id) {
this.originUrl = originUrl;
this.filename = filename;
this.subDir = subDir;
this.metadata = metadata;
this.id = id;
this.isVideo = false;
this.MAX_RETRIES = 3;
this.tmpDir = path_1.default.join('.tmp', [this.id, (0, crypto_1.randomUUID)()].filter((v) => v).join('_'));
this.tmpPlaylistFile = 'playlist.m3u8';
this.chunkLimiter = new bottleneck_1.default({ maxConcurrent: 5 });
/**
* @see https://github.com/HoloArchivists/tslazer/commit/56ba5e11cc4c1a6bab1845e835f7fc4de4babb99
*/
this.ignoreBuffer = Buffer.from([0x49, 0x44, 0x33, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3F, 0x50, 0x52, 0x49, 0x56]);
this.id = this.id || (0, crypto_1.randomUUID)();
this.logger = logger_1.logger.child({ label: `[SpaceDownloader@${this.id}]` });
this.logger.debug('constructor', {
originUrl,
filename,
subDir,
metadata,
});
this.initPaylistFile();
this.initResultFile();
}
initPaylistFile() {
this.playlistFile = path_1.default.join(Util_1.Util.getMediaDir(this.subDir), `${this.filename}.m3u8`);
this.logger.verbose(`Playlist path: "${this.playlistFile}"`);
}
initResultFile() {
const { config } = ConfigManager_1.configManager;
let ext = this.isVideo
? config?.ffmpeg?.video?.extension || 'mp4'
: config?.ffmpeg?.audio?.extension || 'm4a';
ext = ext.replace(/^\./, '');
this.resultFile = path_1.default.join(Util_1.Util.getMediaDir(this.subDir), `${this.filename}.${ext}`);
this.logger.verbose(`Result path: "${this.resultFile}"`);
}
async download() {
this.logger.debug('download', { playlistUrl: this.playlistUrl, originUrl: this.originUrl });
const downloadStart = Date.now();
if (!this.playlistUrl) {
this.playlistUrl = await PeriscopeApi_1.PeriscopeApi.getFinalPlaylistUrl(this.originUrl);
this.logger.info(`Final playlist url: ${this.playlistUrl}`);
}
let data = await axios_1.default.get(this.playlistUrl).then((v) => v.data);
const manifest = m3u8_util_1.M3u8Util.parse(data);
if (manifest.playlists?.length) {
const item = manifest.playlists.sort((a, b) => b.attributes.RESOLUTION.height - a.attributes.RESOLUTION.height || b.attributes.RESOLUTION.width - a.attributes.RESOLUTION.width)[0];
this.playlistUrl = new URL(item.uri, this.playlistUrl).href;
this.logger.info(`Final playlist url: ${this.playlistUrl}`);
data = await axios_1.default.get(this.playlistUrl).then((v) => v.data);
this.isVideo = true;
this.initResultFile();
}
(0, fs_1.mkdirSync)(this.tmpDir, { recursive: true });
Util_1.Util.createMediaDir(this.subDir);
if (this.isVideo) {
await this.spawnFfmpeg(this.playlistUrl);
}
else {
(0, fs_1.writeFileSync)(path_1.default.join(this.tmpDir, this.tmpPlaylistFile), data);
await this.downloadChunks();
await this.spawnFfmpeg(this.tmpPlaylistFile);
}
(0, fs_1.rmSync)(this.tmpDir, { recursive: true, force: true });
const downloadEnd = Date.now();
this.logger.info('Download completed', {
downloadTime: downloadEnd - downloadStart,
downloadStart,
downloadEnd,
});
}
// #region audio
async downloadChunks() {
const m3u8 = (0, fs_1.readFileSync)(path_1.default.join(this.tmpDir, this.tmpPlaylistFile), 'utf8');
const chunks = m3u8.match(/chunk_[\w.]*/g);
if (!chunks.length) {
throw new Error('CHUNK_NOT_FOUND');
}
const chunkBaseUrl = PeriscopeUtil_1.PeriscopeUtil.getChunkPrefix(this.playlistUrl);
await Promise.all(chunks.map((v) => this.chunkLimiter.schedule(() => this.downloadChunk(chunkBaseUrl, v))));
}
async downloadChunk(chunkBaseUrl, chunkName) {
const url = chunkBaseUrl + chunkName;
this.logger.debug(`downloadChunk: ${chunkName}`);
// eslint-disable-next-line no-plusplus
for (let retries = 0; retries <= this.MAX_RETRIES; retries++) {
try {
// eslint-disable-next-line no-await-in-loop
const { data } = await axios_1.default.get(url, { responseType: 'arraybuffer' });
let buffer = Buffer.from(data);
const index = buffer.indexOf(this.ignoreBuffer);
if (index !== -1) {
buffer = buffer.slice(index);
}
(0, fs_1.writeFileSync)(path_1.default.join(this.tmpDir, chunkName), buffer);
return;
}
catch (error) {
if (retries >= this.MAX_RETRIES) {
this.logger.error(`downloadChunk: ${error.message}`, { chunkName, url, retries });
throw error;
}
this.logger.warn(`downloadChunk: ${error.message}`, { chunkName, url, retries });
}
}
}
// #endregion
// #region ffmpeg
async spawnFfmpeg(inp) {
const cmd = 'ffmpeg';
const args = [
// '-protocol_whitelist',
// 'file,https,tls,tcp',
'-i',
inp,
'-seg_max_retry',
'3',
];
if (this.metadata) {
this.logger.debug('File metadata', this.metadata);
Object.keys(this.metadata).forEach((key) => {
const value = this.metadata[key];
if (!value) {
return;
}
args.push('-metadata', `${key}=${value}`);
});
}
const { config } = ConfigManager_1.configManager;
const mediaType = this.isVideo ? 'video' : 'audio';
const mediaArgs = config?.ffmpeg?.[mediaType]?.args;
if (mediaArgs?.length > 0) {
args.push(...mediaArgs);
}
else if (config?.ffmpegArgs?.length) {
args.push(...config.ffmpegArgs);
}
else {
args.push('-c', 'copy');
}
args.push(this.resultFile);
this.logger.verbose(`File is saving to "${this.resultFile}"`);
this.logger.verbose(`${cmd} ${args.join(' ')}`);
const cwd = path_1.default.join(process.cwd(), this.tmpDir);
// https://github.com/nodejs/node/issues/21825
const spawnOptions = {
cwd,
stdio: 'ignore',
detached: false,
windowsHide: true,
};
const cp = process.platform === 'win32'
? (0, child_process_1.spawn)(process.env.comspec, ['/c', cmd, ...args], spawnOptions)
: (0, child_process_1.spawn)(cmd, args, spawnOptions);
// cp.unref()
return new Promise((resolve, reject) => {
cp.once('error', (error) => {
reject(error);
});
cp.once('close', (code) => {
resolve(code);
});
});
}
}
exports.SpaceDownloader = SpaceDownloader;
//# sourceMappingURL=SpaceDownloader.js.map