itch-dl
Version:
Bulk download games from itch.io - TypeScript implementation
217 lines (216 loc) • 8.22 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getJobsForGameJamJson = getJobsForGameJamJson;
exports.getGameJamJson = getGameJamJson;
exports.getJobsForBrowseUrl = getJobsForBrowseUrl;
exports.getJobsForCollectionJson = getJobsForCollectionJson;
exports.getJobsForCreator = getJobsForCreator;
exports.getJobsForItchUrl = getJobsForItchUrl;
exports.getJobsForPath = getJobsForPath;
exports.getJobsForUrlOrPath = getJobsForUrlOrPath;
exports.preprocessJobUrls = preprocessJobUrls;
const node_fs_1 = __importDefault(require("node:fs"));
const cheerio_1 = require("cheerio");
const api_1 = require("./api");
const utils_1 = require("./utils");
const consts_1 = require("./consts");
const keys_1 = require("./keys");
function getJobsForGameJamJson(gameJamJson) {
if (!('jam_games' in gameJamJson)) {
throw new Error('Provided JSON is not a valid itch.io jam JSON.');
}
return gameJamJson.jam_games.map((g) => g.game.url);
}
async function getGameJamJson(jamUrl, client) {
const r = await client.get(jamUrl, false);
if (r.status !== 200) {
throw new utils_1.ItchDownloadError(`Could not download the game jam site: ${r.status} ${r.statusText}`);
}
const jamId = (0, utils_1.getIntAfterMarkerInJson)(r.data, 'I.ViewJam', 'id');
if (jamId === null) {
throw new utils_1.ItchDownloadError('Provided site did not contain the Game Jam ID. Provide the path to the game jam entries JSON file instead.');
}
const r2 = await client.get(`${consts_1.ITCH_URL}/jam/${jamId}/entries.json`);
if (r2.status !== 200) {
throw new utils_1.ItchDownloadError(`Could not download the game jam entries list: ${r2.status} ${r2.statusText}`);
}
return r2.data;
}
async function getJobsForBrowseUrl(url, client) {
let page = 1;
const found = new Set();
while (true) {
const r = await client.get(`${url}.xml?page=${page}`, false);
if (r.status !== 200) {
break;
}
const $ = (0, cheerio_1.load)(r.data, { xmlMode: true });
const items = $('item');
if (items.length < 1) {
break;
}
items.each((_, item) => {
const link = $(item).find('link').text().trim();
if (link) {
found.add(link);
}
});
page += 1;
}
if (found.size === 0) {
throw new utils_1.ItchDownloadError('No game URLs found to download.');
}
return Array.from(found);
}
async function getJobsForCollectionJson(url, client) {
let page = 1;
const found = new Set();
while (true) {
const r = await client.get(url, true, { params: { page }, timeout: 15000 });
if (r.status !== 200) {
break;
}
const data = r.data;
if (data.collection_games.length < 1) {
break;
}
for (const item of data.collection_games) {
found.add(item.game.url);
}
if (data.collection_games.length === data.per_page) {
page += 1;
}
else {
break;
}
}
if (found.size === 0) {
throw new utils_1.ItchDownloadError('No game URLs found to download.');
}
return Array.from(found);
}
async function getJobsForCreator(creator, client) {
const r = await client.get(`https://${consts_1.ITCH_BASE}/profile/${creator}`, false);
if (r.status !== 200) {
throw new utils_1.ItchDownloadError(`Could not fetch the creator page: HTTP ${r.status} ${r.statusText}`);
}
const prefix = `https://${creator}.${consts_1.ITCH_BASE}/`;
const $ = (0, cheerio_1.load)(r.data);
const gameLinks = new Set();
$('a.game_link').each((_, a) => {
const href = $(a).attr('href');
if (href && href.startsWith(prefix)) {
gameLinks.add(href);
}
});
return Array.from(gameLinks).sort();
}
async function getJobsForItchUrl(url, client) {
if (url.startsWith('http://')) {
url = 'https://' + url.slice(7);
}
if (url.startsWith(`https://www.${consts_1.ITCH_BASE}/`)) {
url = consts_1.ITCH_URL + '/' + url.slice(20);
}
const urlObj = new URL(url);
const parts = urlObj.pathname.split('/').filter(x => x.length > 0);
if (urlObj.hostname === consts_1.ITCH_BASE) {
if (parts.length === 0) {
throw new Error('itch-dl cannot download the entirety of itch.io.');
}
const site = parts[0];
if (site === 'jam') {
if (parts.length < 2) {
throw new Error(`Incomplete game jam URL: ${url}`);
}
const clean = `${consts_1.ITCH_URL}/jam/${parts[1]}`;
const jamJson = await getGameJamJson(clean, client);
return getJobsForGameJamJson(jamJson);
}
else if (consts_1.ITCH_BROWSER_TYPES.includes(site)) {
const clean = [consts_1.ITCH_URL, ...parts].join('/');
return await getJobsForBrowseUrl(clean, client);
}
else if (site === 'b' || site === 'bundle') {
throw new Error('itch-dl cannot download bundles yet.');
}
else if (['j', 'jobs'].includes(site)) {
throw new Error('itch-dl cannot download a job.');
}
else if (['t', 'board', 'community'].includes(site)) {
throw new Error('itch-dl cannot download forums.');
}
else if (site === 'profile') {
if (parts.length >= 2) {
return await getJobsForCreator(parts[1], client);
}
throw new Error('itch-dl expects a username in profile links.');
}
else if (site === 'my-purchases') {
return await (0, keys_1.getOwnedGames)(client);
}
else if (site === 'c') {
const collectionId = parts[1];
const clean = `${consts_1.ITCH_API}/collections/${collectionId}/collection-games`;
return await getJobsForCollectionJson(clean, client);
}
throw new Error(`itch-dl does not understand "${site}" URLs.`);
}
else if (urlObj.hostname.endsWith(`.${consts_1.ITCH_BASE}`)) {
if (parts.length === 0) {
return await getJobsForCreator(urlObj.hostname.split('.')[0], client);
}
return [`https://${urlObj.hostname}/${parts[0]}`];
}
else {
throw new Error(`Unknown domain: ${urlObj.hostname}`);
}
}
function getJobsForPath(p) {
try {
const jsonData = JSON.parse(node_fs_1.default.readFileSync(p, 'utf8'));
if (jsonData && jsonData.jam_games) {
return getJobsForGameJamJson(jsonData);
}
}
catch {
// ignore
}
const lines = node_fs_1.default.readFileSync(p, 'utf8').split(/\r?\n/);
const urlList = lines.filter(l => l.startsWith('https://') || l.startsWith('http://'));
if (urlList.length > 0) {
return urlList;
}
throw new Error('File format is unknown - cannot read URLs to download.');
}
async function getJobsForUrlOrPath(pathOrUrl, settings) {
pathOrUrl = pathOrUrl.trim();
if (pathOrUrl.startsWith('http://')) {
pathOrUrl = 'https://' + pathOrUrl.slice(7);
}
if (pathOrUrl.startsWith('https://')) {
const client = new api_1.ItchApiClient(settings.apiKey, settings.userAgent);
return await getJobsForItchUrl(pathOrUrl, client);
}
else if (node_fs_1.default.existsSync(pathOrUrl) && node_fs_1.default.statSync(pathOrUrl).isFile()) {
return getJobsForPath(pathOrUrl);
}
throw new Error(`Cannot handle path or URL: ${pathOrUrl}`);
}
function preprocessJobUrls(jobs, settings) {
const cleaned = new Set();
for (const baseJob of jobs) {
const job = baseJob.trim();
if ((0, utils_1.shouldSkipItemByGlob)('URL', job, settings.filterUrlsGlob)) {
continue;
}
if ((0, utils_1.shouldSkipItemByRegex)('URL', job, settings.filterUrlsRegex)) {
continue;
}
cleaned.add(job);
}
return Array.from(cleaned);
}