twspace-crawler
Version:
Script to monitor & download Twitter Spaces 24/7
129 lines • 5.84 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SpaceDownloader = void 0;
const axios_1 = __importDefault(require("axios"));
const bottleneck_1 = __importDefault(require("bottleneck"));
const child_process_1 = require("child_process");
const crypto_1 = require("crypto");
const fs_1 = require("fs");
const path_1 = __importDefault(require("path"));
const PeriscopeApi_1 = require("../apis/PeriscopeApi");
const logger_1 = require("../logger");
const PeriscopeUtil_1 = require("../utils/PeriscopeUtil");
const Util_1 = require("../utils/Util");
const ConfigManager_1 = require("./ConfigManager");
class SpaceDownloader {
constructor(originUrl, filename, subDir = '', metadata, id) {
this.originUrl = originUrl;
this.filename = filename;
this.subDir = subDir;
this.metadata = metadata;
this.id = id;
this.tmpDir = path_1.default.join('.tmp', this.id || (0, crypto_1.randomUUID)());
this.tmpPlaylistFile = 'playlist.m3u8';
this.chunkLimiter = new bottleneck_1.default({ maxConcurrent: 5 });
/**
* @see https://github.com/HoloArchivists/tslazer/commit/56ba5e11cc4c1a6bab1845e835f7fc4de4babb99
*/
this.ignoreBuffer = Buffer.from([0x49, 0x44, 0x33, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3F, 0x50, 0x52, 0x49, 0x56]);
this.id = this.id || (0, crypto_1.randomUUID)();
this.logger = logger_1.logger.child({ label: `[SpaceDownloader@${this.id}]` });
this.logger.debug('constructor', {
originUrl, filename, subDir, metadata,
});
this.playlistFile = path_1.default.join(Util_1.Util.getMediaDir(subDir), `${filename}.m3u8`);
this.audioFile = path_1.default.join(Util_1.Util.getMediaDir(subDir), `${filename}.m4a`);
this.logger.verbose(`Playlist path: "${this.playlistFile}"`);
this.logger.verbose(`Audio path: "${this.audioFile}"`);
}
async download() {
this.logger.debug('download', { playlistUrl: this.playlistUrl, originUrl: this.originUrl });
if (!this.playlistUrl) {
this.playlistUrl = await PeriscopeApi_1.PeriscopeApi.getFinalPlaylistUrl(this.originUrl);
this.logger.info(`Final playlist url: ${this.playlistUrl}`);
}
(0, fs_1.mkdirSync)(this.tmpDir, { recursive: true });
await this.downloadPlaylist();
await this.downloadChunks();
Util_1.Util.createMediaDir(this.subDir);
await this.spawnFfmpeg();
(0, fs_1.rmSync)(this.tmpDir, { recursive: true, force: true });
}
async spawnFfmpeg() {
const cmd = 'ffmpeg';
const args = [
// '-protocol_whitelist',
// 'file,https,tls,tcp',
'-i',
this.tmpPlaylistFile,
'-c',
'copy',
];
if (this.metadata) {
this.logger.debug('Audio metadata', this.metadata);
Object.keys(this.metadata).forEach((key) => {
const value = this.metadata[key];
if (!value) {
return;
}
args.push('-metadata', `${key}=${value}`);
});
}
const { config } = ConfigManager_1.configManager;
if (config?.ffmpegArgs?.length) {
args.push(...config.ffmpegArgs);
}
args.push(this.audioFile);
this.logger.verbose(`Audio is saving to "${this.audioFile}"`);
this.logger.verbose(`${cmd} ${args.join(' ')}`);
const cwd = path_1.default.join(process.cwd(), this.tmpDir);
// https://github.com/nodejs/node/issues/21825
const spawnOptions = {
cwd,
stdio: 'ignore',
detached: false,
windowsHide: true,
};
const cp = process.platform === 'win32'
? (0, child_process_1.spawn)(process.env.comspec, ['/c', cmd, ...args], spawnOptions)
: (0, child_process_1.spawn)(cmd, args, spawnOptions);
// cp.unref()
return new Promise((resolve, reject) => {
cp.once('error', (error) => {
reject(error);
});
cp.once('close', (code) => {
resolve(code);
});
});
}
async downloadPlaylist() {
const { data } = await axios_1.default.get(this.playlistUrl);
(0, fs_1.writeFileSync)(path_1.default.join(this.tmpDir, this.tmpPlaylistFile), data);
}
async downloadChunks() {
const m3u8 = (0, fs_1.readFileSync)(path_1.default.join(this.tmpDir, this.tmpPlaylistFile), 'utf8');
const chunks = m3u8.match(/chunk_[\w.]*/g);
if (!chunks.length) {
throw new Error('CHUNK_NOT_FOUND');
}
const chunkBaseUrl = PeriscopeUtil_1.PeriscopeUtil.getChunkPrefix(this.playlistUrl);
await Promise.all(chunks.map((v) => this.chunkLimiter.schedule(() => this.downloadChunk(chunkBaseUrl, v))));
}
async downloadChunk(chunkBaseUrl, chunkName) {
const url = chunkBaseUrl + chunkName;
this.logger.debug(`downloadChunk: ${chunkName}`);
const { data } = await axios_1.default.get(url, { responseType: 'arraybuffer' });
let buffer = Buffer.from(data);
const index = buffer.indexOf(this.ignoreBuffer);
if (index !== -1) {
buffer = buffer.slice(index);
}
(0, fs_1.writeFileSync)(path_1.default.join(this.tmpDir, chunkName), buffer);
}
}
exports.SpaceDownloader = SpaceDownloader;
//# sourceMappingURL=SpaceDownloader.js.map