itch-dl
Version:
Bulk download games from itch.io - TypeScript implementation
436 lines (435 loc) • 16.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.GameDownloader = exports.TARGET_PATHS = void 0;
exports.driveDownloads = driveDownloads;
const node_fs_1 = __importDefault(require("node:fs"));
const node_path_1 = __importDefault(require("node:path"));
const cheerio_1 = require("cheerio");
const adm_zip_1 = __importDefault(require("adm-zip"));
const tar = __importStar(require("tar"));
const cli_progress_1 = require("cli-progress");
const api_1 = require("./api");
const utils_1 = require("./utils");
const consts_1 = require("./consts");
const infobox_1 = require("./infobox");
exports.TARGET_PATHS = {
site: 'site.html',
cover: 'cover',
metadata: 'metadata.json',
files: 'files',
screenshots: 'screenshots',
};
class GameDownloader {
constructor(settings, keys) {
this.settings = settings;
this.downloadKeys = keys;
this.client = new api_1.ItchApiClient(settings.apiKey, settings.userAgent);
}
static getRatingJson(site) {
const scripts = site('script[type="application/ld+json"]');
for (let i = 0; i < scripts.length; i++) {
try {
const json = JSON.parse(site(scripts[i]).text().trim());
if (json['@type'] === 'Product') {
return json;
}
}
catch {
continue;
}
}
return null;
}
static getMeta(site, selector, attr = 'content') {
const node = site(`meta[${selector}]`);
if (node.length === 0) {
return null;
}
return node.attr(attr) || null;
}
async getGameId(url, site) {
let gameId = null;
const itchPath = GameDownloader.getMeta(site, 'name="itch:path"');
if (itchPath) {
const parts = itchPath.split('/');
const last = parts[parts.length - 1];
gameId = parseInt(last, 10);
}
if (!gameId) {
const scripts = site('script[type="text/javascript"]');
for (let i = 0; i < scripts.length; i++) {
const text = site(scripts[i]).text().trim();
if (text.includes('I.ViewGame')) {
const v = (0, utils_1.getIntAfterMarkerInJson)(text, 'I.ViewGame', 'id');
if (v !== null) {
gameId = v;
break;
}
}
}
}
if (!gameId) {
const dataUrl = url.replace(/\/+$/, '') + '/data.json';
const r = await this.client.get(dataUrl, false);
if (r.status === 200) {
try {
const data = r.data;
if (data.errors) {
throw new utils_1.ItchDownloadError(`Game data fetching failed for ${url}: ${data.errors}`);
}
if (data.id) {
gameId = parseInt(data.id, 10);
}
}
catch { }
}
}
if (!gameId) {
throw new utils_1.ItchDownloadError(`Could not get the Game ID for URL: ${url}`);
}
return gameId;
}
extractMetadata(gameId, url, site) {
const ratingJson = GameDownloader.getRatingJson(site);
const title = ratingJson ? ratingJson.name : null;
let description = GameDownloader.getMeta(site, 'property="og:description"');
if (!description) {
description = GameDownloader.getMeta(site, 'name="description"');
}
const screenshots = [];
const screenshotsNode = site('div.screenshot_list');
screenshotsNode.find('a').each((_, a) => {
const href = site(a).attr('href');
if (href) {
screenshots.push(href);
}
});
const metadata = {
game_id: gameId,
title: title || site('h1.game_title').text().trim(),
url,
cover_url: GameDownloader.getMeta(site, 'property="og:image"'),
screenshots,
description: description || undefined,
};
const infoboxDiv = site('div.game_info_panel_widget');
if (infoboxDiv.length) {
const infobox = (0, infobox_1.parseInfobox)(infoboxDiv);
for (const dt of ['created_at', 'updated_at', 'released_at', 'published_at']) {
if (infobox[dt]) {
metadata[dt] = infobox[dt].toISOString();
delete infobox[dt];
}
}
if ('author' in infobox) {
metadata.author = infobox.author.author;
metadata.author_url = infobox.author.author_url;
delete infobox.author;
}
if ('authors' in infobox && !metadata.author) {
metadata.author = 'Multiple authors';
metadata.author_url = `https://${new URL(url).hostname}`;
}
metadata.extra = infobox;
}
const aggRating = ratingJson ? ratingJson.aggregateRating : null;
if (aggRating) {
try {
metadata.rating = {
average: parseFloat(aggRating.ratingValue),
votes: aggRating.ratingCount,
};
}
catch {
// ignore
}
}
return metadata;
}
getCredentials(title, gameId) {
const creds = {};
if (this.downloadKeys[gameId]) {
creds['download_key_id'] = this.downloadKeys[gameId];
console.debug('Got credentials for', title, creds);
}
return creds;
}
static async getDecompressedContentSize(targetPath) {
try {
const zip = new adm_zip_1.default(targetPath);
const entries = zip.getEntries().filter((e) => !e.isDirectory);
if (entries.length > 0) {
return entries.reduce((sum, ent) => sum + ent.header.size, 0);
}
}
catch {
// not a zip file
}
try {
let total = 0;
await tar.t({
file: targetPath,
onentry: (entry) => {
if (entry.type === 'File') {
total += entry.size;
}
},
});
return total === 0 ? null : total;
}
catch {
// not a tar file
}
return null;
}
async downloadFile(url, downloadPath, credentials) {
try {
const res = await this.client.get(url, true, {
responseType: 'stream',
data: credentials,
});
if (downloadPath) {
await new Promise((resolve, reject) => {
const writer = node_fs_1.default.createWriteStream(downloadPath);
res.data.pipe(writer);
writer.on('finish', resolve);
writer.on('error', reject);
});
}
return res.request.res.responseUrl || url; // final URL
}
catch (e) {
throw new utils_1.ItchDownloadError(`Unrecoverable download error: ${e}`);
}
}
async downloadFileByUploadId(uploadId, downloadPath, credentials) {
return this.downloadFile(`/uploads/${uploadId}/download`, downloadPath, credentials);
}
async download(url, skipDownloaded = true) {
const match = url.match(consts_1.ITCH_GAME_URL_REGEX);
if (!match || !match.groups) {
return {
url,
success: false,
errors: [`Game URL is invalid: ${url} - please file a new issue.`],
external_urls: [],
};
}
const author = match.groups.author;
const game = match.groups.game;
const downloadPath = node_path_1.default.join(this.settings.downloadTo ?? '.', author, game);
node_fs_1.default.mkdirSync(downloadPath, { recursive: true });
const paths = {};
for (const [k, v] of Object.entries(exports.TARGET_PATHS)) {
paths[k] = node_path_1.default.join(downloadPath, v);
}
if (node_fs_1.default.existsSync(paths['metadata']) && skipDownloaded) {
console.info('Skipping already-downloaded game for URL:', url);
return { url, success: true, errors: ['Game already downloaded.'], external_urls: [] };
}
let siteHtml = '';
try {
const r = await this.client.get(url, false);
siteHtml = r.data;
}
catch (e) {
return {
url,
success: false,
errors: [`Could not download the game site for ${url}: ${e}`],
external_urls: [],
};
}
const $ = (0, cheerio_1.load)(siteHtml);
let gameId;
let metadata;
try {
gameId = await this.getGameId(url, $);
metadata = this.extractMetadata(gameId, url, $);
}
catch (e) {
return { url, success: false, errors: [String(e)], external_urls: [] };
}
const title = metadata.title || game;
const credentials = this.getCredentials(title, gameId);
let gameUploads;
try {
const uploadsReq = await this.client.get(`/games/${gameId}/uploads`, true, {
data: credentials,
timeout: 15000,
});
gameUploads = uploadsReq.data.uploads;
}
catch (e) {
return {
url,
success: false,
errors: [`Could not fetch game uploads for ${title}: ${e}`],
external_urls: [],
};
}
const externalUrls = [];
const errors = [];
node_fs_1.default.mkdirSync(paths['files'], { recursive: true });
for (const upload of gameUploads) {
if (!('id' in upload &&
'filename' in upload &&
'type' in upload &&
'traits' in upload &&
'storage' in upload)) {
errors.push(`Upload metadata incomplete: ${JSON.stringify(upload)}`);
continue;
}
const uploadId = upload.id;
const fileName = upload.filename;
const fileType = upload.type;
const fileTraits = upload.traits;
const expectedSize = upload.size;
const uploadIsExternal = upload.storage === 'external';
if (this.settings.filterFilesType && !this.settings.filterFilesType.includes(fileType)) {
console.info(`File '${fileName}' has ignored type '${fileType}', skipping`);
continue;
}
if (this.settings.filterFilesPlatform &&
fileType === 'default' &&
!fileTraits.some(t => this.settings.filterFilesPlatform.includes(t))) {
console.info(`File '${fileName}' not for requested platforms, skipping`);
continue;
}
if ((0, utils_1.shouldSkipItemByGlob)('File', fileName, this.settings.filterFilesGlob)) {
continue;
}
if ((0, utils_1.shouldSkipItemByRegex)('File', fileName, this.settings.filterFilesRegex)) {
continue;
}
const targetPath = uploadIsExternal ? null : node_path_1.default.join(paths['files'], fileName);
try {
const targetUrl = await this.downloadFileByUploadId(uploadId, targetPath, credentials);
if (uploadIsExternal) {
externalUrls.push(targetUrl);
continue;
}
const stat = node_fs_1.default.statSync(targetPath);
const downloadedSize = stat.size;
let contentSize = null;
if (expectedSize !== undefined && downloadedSize !== expectedSize) {
contentSize = await GameDownloader.getDecompressedContentSize(targetPath);
if (contentSize !== expectedSize) {
errors.push(`Downloaded file size is ${downloadedSize} (content ${contentSize}), expected ${expectedSize} for upload ${JSON.stringify(upload)}`);
}
}
}
catch (e) {
errors.push(`Download failed for upload ${JSON.stringify(upload)}: ${e}`);
}
}
if (this.settings.mirrorWeb) {
node_fs_1.default.mkdirSync(paths['screenshots'], { recursive: true });
for (const screenshot of metadata.screenshots) {
if (!screenshot) {
continue;
}
const fileName = node_path_1.default.basename(screenshot);
try {
await this.downloadFile(screenshot, node_path_1.default.join(paths['screenshots'], fileName), {});
}
catch (e) {
errors.push(`Screenshot download failed: ${e}`);
}
}
}
if (metadata.cover_url) {
try {
const ext = node_path_1.default.extname(metadata.cover_url);
await this.downloadFile(metadata.cover_url, paths['cover'] + ext, {});
}
catch (e) {
errors.push(`Cover art download failed: ${e}`);
}
}
node_fs_1.default.writeFileSync(paths['site'], (0, cheerio_1.load)(siteHtml).html() || '');
node_fs_1.default.writeFileSync(paths['metadata'], JSON.stringify(metadata, null, 4));
return { url, success: errors.length === 0, errors, external_urls: externalUrls };
}
}
exports.GameDownloader = GameDownloader;
async function driveDownloads(jobs, settings, keys) {
const downloader = new GameDownloader(settings, keys);
const results = new Array(jobs.length);
const bar = new cli_progress_1.SingleBar({}, cli_progress_1.Presets.shades_classic);
bar.start(jobs.length, 0);
let index = 0;
const workers = [];
const threads = Math.max(1, settings.parallel);
const worker = async () => {
while (true) {
if (index >= jobs.length) {
return;
}
const current = index++;
const job = jobs[current];
const res = await downloader.download(job);
results[current] = res;
bar.increment();
}
};
for (let i = 0; i < threads; i++) {
workers.push(worker());
}
await Promise.all(workers);
bar.stop();
console.log('Download complete!');
for (const r of results) {
if (!r.errors.length && !r.external_urls.length) {
continue;
}
if (r.success) {
console.log(`\nNotes for ${r.url}:`);
}
else {
console.log(`\nDownload failed for ${r.url}:`);
}
for (const e of r.errors) {
console.log(`- ${e}`);
}
for (const u of r.external_urls) {
console.log(`- External download URL (download manually!): ${u}`);
}
}
}