itch-dl
Version:
Bulk download games from itch.io - TypeScript implementation
195 lines (194 loc) • 9.14 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const node_test_1 = __importDefault(require("node:test"));
const node_assert_1 = __importDefault(require("node:assert"));
const cheerio_1 = require("cheerio");
const node_fs_1 = __importDefault(require("node:fs"));
const node_path_1 = __importDefault(require("node:path"));
const node_os_1 = __importDefault(require("node:os"));
const adm_zip_1 = __importDefault(require("adm-zip"));
const tar = __importStar(require("tar"));
const downloader_1 = require("../src/downloader");
const config_1 = require("../src/config");
const downloader_2 = require("../src/downloader");
const cli_progress_1 = require("cli-progress");
(0, node_test_1.default)('GameDownloader.getRatingJson and getMeta', () => {
const html = `<script type="application/ld+json">{"@type":"Product","name":"G"}</script>` +
`<meta property="og:image" content="img.png">`;
const $ = (0, cheerio_1.load)(html);
const json = downloader_1.GameDownloader.getRatingJson($);
node_assert_1.default.deepStrictEqual(json, { '@type': 'Product', name: 'G' });
node_assert_1.default.strictEqual(downloader_1.GameDownloader.getMeta($, 'property="og:image"'), 'img.png');
});
(0, node_test_1.default)('GameDownloader.getGameId from meta and script', async () => {
const settings = { ...config_1.DEFAULT_SETTINGS };
const gd = new downloader_1.GameDownloader(settings, {});
const site1 = (0, cheerio_1.load)('<meta name="itch:path" content="games/5"/>');
node_assert_1.default.strictEqual(await gd.getGameId('https://a.itch.io/game', site1), 5);
const site2 = (0, cheerio_1.load)('<script type="text/javascript">I.ViewGame({"id":7})</script>');
node_assert_1.default.strictEqual(await gd.getGameId('https://a.itch.io/game', site2), 7);
});
(0, node_test_1.default)('GameDownloader.extractMetadata basic', () => {
const html = `\n <html><head>
<script type="application/ld+json">{"@type":"Product","name":"My Game","aggregateRating":{"ratingValue":"4.5","ratingCount":10}}</script>
<meta property="og:image" content="cover.png">
<meta property="og:description" content="desc">
</head><body>
<h1 class="game_title">My Game</h1>
<div class="screenshot_list"><a href="sc1.png"></a></div>
<div class="game_info_panel_widget"><table>
<tr><td>Author</td><td><a href="https://dev">Dev</a></td></tr>
<tr><td>Published</td><td><abbr title="01 January 2024 @ 15:00 UTC"></abbr></td></tr>
</table></div>
</body></html>`;
const $ = (0, cheerio_1.load)(html);
const gd = new downloader_1.GameDownloader({ ...config_1.DEFAULT_SETTINGS }, {});
const meta = gd.extractMetadata(1, 'https://a.itch.io/game', $);
node_assert_1.default.strictEqual(meta.title, 'My Game');
node_assert_1.default.strictEqual(meta.cover_url, 'cover.png');
node_assert_1.default.deepStrictEqual(meta.screenshots, ['sc1.png']);
node_assert_1.default.strictEqual(meta.author, 'Dev');
node_assert_1.default.strictEqual(meta.author_url, 'https://dev');
node_assert_1.default.strictEqual(meta.published_at, new Date('01 January 2024 15:00 UTC').toISOString());
node_assert_1.default.ok(meta.extra && Object.keys(meta.extra).length === 0);
});
(0, node_test_1.default)('driveDownloads runs downloads concurrently and reports progress', async () => {
const origDownload = downloader_1.GameDownloader.prototype.download;
let active = 0;
let maxActive = 0;
let calls = 0;
downloader_1.GameDownloader.prototype.download = async function (url) {
active++;
calls++;
if (active > maxActive) {
maxActive = active;
}
await new Promise(res => setTimeout(res, 50));
active--;
return { url, success: true, errors: [], external_urls: [] };
};
const proto = cli_progress_1.SingleBar.prototype;
const origStart = proto.start;
const origIncrement = proto.increment;
const origStop = proto.stop;
const events = [];
proto.start = function (total, start) {
events.push(['start', total, start]);
};
proto.increment = function () {
events.push(['inc']);
};
proto.stop = function () {
events.push(['stop']);
};
const settings = { ...config_1.DEFAULT_SETTINGS, parallel: 2 };
await (0, downloader_2.driveDownloads)(['u1', 'u2', 'u3'], settings, {});
downloader_1.GameDownloader.prototype.download = origDownload;
proto.start = origStart;
proto.increment = origIncrement;
proto.stop = origStop;
node_assert_1.default.strictEqual(calls, 3);
node_assert_1.default.ok(maxActive > 1);
node_assert_1.default.deepStrictEqual(events, [['start', 3, 0], ['inc'], ['inc'], ['inc'], ['stop']]);
});
(0, node_test_1.default)('driveDownloads updates progress and concurrency', async () => {
const cliProgress = require('cli-progress');
const OriginalBar = cliProgress.SingleBar;
const events = {
increments: 0,
stopped: false,
};
class FakeBar {
start(...args) {
events.start = args;
}
increment() {
events.increments++;
}
stop() {
events.stopped = true;
}
}
cliProgress.SingleBar = class {
constructor() {
return new FakeBar();
}
};
const origDownload = downloader_1.GameDownloader.prototype.download;
const starts = [];
downloader_1.GameDownloader.prototype.download = async function (url) {
starts.push(Date.now());
await new Promise(res => setTimeout(res, 50));
return { url, success: true, errors: [], external_urls: [] };
};
const settings = { ...config_1.DEFAULT_SETTINGS, parallel: 2 };
const t0 = Date.now();
await (0, downloader_2.driveDownloads)(['a', 'b', 'c'], settings, {});
const duration = Date.now() - t0;
downloader_1.GameDownloader.prototype.download = origDownload;
cliProgress.SingleBar = OriginalBar;
node_assert_1.default.strictEqual(starts.length, 3);
node_assert_1.default.ok(starts[1] - starts[0] < 40);
// Be more lenient with timing on Windows due to different performance characteristics
const maxDuration = process.platform === 'win32' ? 200 : 120;
node_assert_1.default.ok(duration < maxDuration);
node_assert_1.default.deepStrictEqual(events.start, [3, 0]);
node_assert_1.default.strictEqual(events.increments, 3);
node_assert_1.default.ok(events.stopped);
});
(0, node_test_1.default)('GameDownloader.getDecompressedContentSize for zip and tar', async () => {
const tmp = node_fs_1.default.mkdtempSync(node_path_1.default.join(node_os_1.default.tmpdir(), 'itch-dl-test-'));
const zipPath = node_path_1.default.join(tmp, 't.zip');
const tarPath = node_path_1.default.join(tmp, 't.tar');
const f1 = node_path_1.default.join(tmp, 'a.txt');
const f2 = node_path_1.default.join(tmp, 'b.txt');
node_fs_1.default.writeFileSync(f1, 'abc');
node_fs_1.default.writeFileSync(f2, 'de');
const expected = 5;
const zip = new adm_zip_1.default();
zip.addLocalFile(f1);
zip.addLocalFile(f2);
zip.writeZip(zipPath);
await tar.c({ cwd: tmp, file: tarPath }, ['a.txt', 'b.txt']);
const zsize = await downloader_1.GameDownloader.getDecompressedContentSize(zipPath);
const tsize = await downloader_1.GameDownloader.getDecompressedContentSize(tarPath);
node_assert_1.default.strictEqual(zsize, expected);
node_assert_1.default.strictEqual(tsize, expected);
node_fs_1.default.rmSync(tmp, { recursive: true, force: true });
});