igir
Version:
🕹 A zero-setup ROM collection manager that sorts, filters, extracts or archives, patches, and reports on collections of any size on any OS.
295 lines (294 loc) • 13.8 kB
JavaScript
import fs from 'node:fs';
import path from 'node:path';
import util from 'node:util';
import { parse } from '@gplane/cue';
import Package from '../../globals/package.js';
import ArrayPoly from '../../polyfill/arrayPoly.js';
import Game from '../../types/dats/game.js';
import Header from '../../types/dats/logiqx/header.js';
import LogiqxDAT from '../../types/dats/logiqx/logiqxDat.js';
import ROM from '../../types/dats/rom.js';
import ArchiveEntry from '../../types/files/archives/archiveEntry.js';
import Module from '../module.js';
/**
* If no {@link DAT}s are provided, implicitly create some. A {@link DAT} will be created for every
* subdirectory that contains files, and {@link Game}s will be named after each file's extracted
* path (without the extension).
*/
export default class DATGameInferrer extends Module {
static DEFAULT_DAT_NAME = Package.NAME;
options;
constructor(options, progressBar) {
super(progressBar, DATGameInferrer.name);
this.options = options;
}
/**
* Infer {@link Game}s from input files.
*/
async infer(romFiles) {
this.progressBar.logTrace(`inferring DATs for ${romFiles.length.toLocaleString()} ROM${romFiles.length === 1 ? '' : 's'}`);
const normalizedInputPaths = this.options
.getInputPaths()
// Try to strip out glob patterns
.map((inputPath) => inputPath.replace(/([\\/][?*]+)+$/, ''));
const inputPathsToRomFiles = romFiles.reduce((map, file) => {
const normalizedPath = path.normalize(file.getFilePath());
const matchedInputPaths = normalizedInputPaths
// `.filter()` rather than `.find()` because a file can be found in overlapping input paths,
// therefore it should be counted in both
.filter((inputPath) => normalizedPath.startsWith(inputPath));
(matchedInputPaths.length > 0
? matchedInputPaths
: [DATGameInferrer.DEFAULT_DAT_NAME]).forEach((inputPath) => {
if (map.has(inputPath)) {
map.get(inputPath)?.push(file);
}
else {
map.set(inputPath, [file]);
}
});
return map;
}, new Map());
this.progressBar.logTrace(`inferred ${inputPathsToRomFiles.size.toLocaleString()} DAT${inputPathsToRomFiles.size === 1 ? '' : 's'}`);
const dats = await Promise.all([...inputPathsToRomFiles.entries()].map(async ([inputPath, datRomFiles]) => this.createDAT(inputPath, datRomFiles)));
this.progressBar.logTrace('done inferring DATs');
return dats;
}
async createDAT(inputPath, romFiles) {
let remainingRomFiles = DATGameInferrer.enrichLikeFiles(romFiles);
let gameNamesToRomFiles = [];
// For each inference strategy
const inferFunctions = [
this.inferArchiveEntries.bind(this),
this.inferBinCueFiles.bind(this),
this.inferGdiFiles.bind(this),
this.inferRawFiles.bind(this),
];
for (const inferFunction of inferFunctions) {
// Infer the games and their files
const result = await inferFunction.bind(this)(remainingRomFiles);
// Update the list of results
gameNamesToRomFiles = [...gameNamesToRomFiles, ...result];
// Remove the consumed files from further inference
const consumedFiles = new Set(result.flatMap(([, resultFiles]) => resultFiles).map((file) => file.toString()));
remainingRomFiles = remainingRomFiles.filter((file) => !consumedFiles.has(file.toString()));
}
const games = gameNamesToRomFiles
.map(([gameName, gameRomFiles]) => {
const roms = gameRomFiles
.map((romFile) => new ROM({
name: romFile.getExtractedFilePath(),
size: romFile.getSize(),
crc32: romFile.getCrc32(),
md5: romFile.getMd5(),
sha1: romFile.getSha1(),
sha256: romFile.getSha256(),
}))
.filter(ArrayPoly.filterUniqueMapped((rom) => rom.getName()))
.sort((a, b) => a.getName().localeCompare(b.getName()));
return new Game({
name: gameName,
description: gameName,
roms: roms,
dir2datSource: gameRomFiles
.map((romFile) => romFile.getFilePath())
.reduce(ArrayPoly.reduceUnique(), [])
.sort()
.join(', '),
});
})
// Filter out duplicate games
.filter(ArrayPoly.filterUniqueMapped((game) => game.hashCode()));
const datName = path.basename(inputPath);
const header = new Header({
name: datName,
description: datName,
});
return new LogiqxDAT({ header, games });
}
/**
* Different types of archives will return different checksums when quick scanning. This will
* result in files that are actually the same having different hash codes.
* Look for files that are the same, combine all known checksums, and enrich files with all
* known checksum information.
*/
static enrichLikeFiles(files) {
const crc32Map = this.combineLikeChecksums(files, (file) => file.getCrc32() !== undefined && file.getSize() > 0
? `${file.getCrc32()}|${file.getSize()}`
: undefined);
const md5Map = this.combineLikeChecksums(files, (file) => file.getMd5());
const sha1Map = this.combineLikeChecksums(files, (file) => file.getSha1());
const sha256Map = this.combineLikeChecksums(files, (file) => file.getSha256());
return files.map((file) => {
let enrichedFile = file;
[
crc32Map.get(`${file.getCrc32()}|${file.getSize()}`),
md5Map.get(file.getMd5() ?? ''),
sha1Map.get(file.getSha1() ?? ''),
sha256Map.get(file.getSha256() ?? ''),
]
.filter((checksumProps) => checksumProps !== undefined)
.forEach((checksumProps) => {
enrichedFile = enrichedFile.withProps(checksumProps);
});
return enrichedFile;
});
}
static combineLikeChecksums(files, keyFunc) {
const crc32Map = files.reduce((map, romFile) => {
const key = keyFunc(romFile);
if (key === undefined) {
return map;
}
if (map.has(key)) {
map.get(key)?.push(romFile);
}
else {
map.set(key, [romFile]);
}
return map;
}, new Map());
return new Map([...crc32Map].map(([key, romFiles]) => {
const checksums = {};
romFiles.forEach((romFile) => {
checksums.crc32 = romFile.getCrc32() ?? checksums.crc32;
checksums.md5 = romFile.getMd5() ?? checksums.md5;
checksums.sha1 = romFile.getSha1() ?? checksums.sha1;
checksums.sha256 = romFile.getSha256() ?? checksums.sha256;
});
return [key, checksums];
}));
}
static getGameName(file) {
// Assume the game name is the filename
let fileName = file.getExtractedFilePath();
if (file instanceof ArchiveEntry) {
// If the file is from an archive, assume the game name is the archive's filename
fileName = file.getArchive().getFilePath();
// If the file is using its correct extension, then slice it off and
// return the result as the game name
const extIdx = fileName.lastIndexOf(file.getArchive().getExtension());
if (extIdx !== -1) {
return path.basename(fileName.slice(0, extIdx)).trim();
}
}
return (path
.basename(fileName)
// Chop off the extension
.replace(/(\.[a-z0-9]+)+$/, '')
.trim());
}
inferArchiveEntries(romFiles) {
this.progressBar.logTrace(`inferring games from archives from ${romFiles.length.toLocaleString()} file${romFiles.length === 1 ? '' : 's'}`);
// For archives, assume the entire archive is one game
const archivePathsToArchiveEntries = romFiles
.filter((file) => file instanceof ArchiveEntry)
.reduce((map, file) => {
const key = `${file.getFilePath()}|${file.getArchive().constructor.name}`;
if (map.has(key)) {
map.get(key)?.push(file);
}
else {
map.set(key, [file]);
}
return map;
}, new Map());
const results = [...archivePathsToArchiveEntries.values()].map((archiveEntries) => {
const gameName = DATGameInferrer.getGameName(archiveEntries[0]);
return [gameName, archiveEntries];
});
this.progressBar.logTrace(`inferred ${results.length.toLocaleString()} games from archives`);
return results;
}
async inferBinCueFiles(romFiles) {
const rawFiles = romFiles.filter((file) => !(file instanceof ArchiveEntry));
this.progressBar.logTrace(`inferring games from cue files from ${rawFiles.length.toLocaleString()} non-archive${rawFiles.length === 1 ? '' : 's'}`);
const rawFilePathsToFiles = rawFiles.reduce((map, file) => {
map.set(file.getFilePath(), file);
return map;
}, new Map());
const results = (await Promise.all(rawFiles
.filter((file) => file.getExtractedFilePath().toLowerCase().endsWith('.cue'))
.map(async (cueFile) => {
try {
const cueData = await util.promisify(fs.readFile)(cueFile.getFilePath());
const cueSheet = parse(cueData.toString(), {
fatal: true,
}).sheet;
const binFiles = cueSheet.files
.map((binFile) => path.join(path.dirname(cueFile.getFilePath()), binFile.name))
.map((binFilePath) => rawFilePathsToFiles.get(binFilePath))
.filter((file) => file !== undefined);
if (binFiles.length === 0) {
return undefined;
}
const gameName = DATGameInferrer.getGameName(cueFile);
return [gameName, [cueFile, ...binFiles]];
}
catch {
return undefined;
}
}))).filter((result) => result !== undefined);
this.progressBar.logTrace(`inferred ${results.length.toLocaleString()} games from cue files`);
return results;
}
async inferGdiFiles(romFiles) {
const rawFiles = romFiles.filter((file) => !(file instanceof ArchiveEntry));
this.progressBar.logTrace(`inferring games from gdi files from ${rawFiles.length.toLocaleString()} non-archive${rawFiles.length === 1 ? '' : 's'}`);
const rawFilePathsToFiles = rawFiles.reduce((map, file) => {
map.set(file.getFilePath(), file);
return map;
}, new Map());
const results = (await Promise.all(rawFiles
.filter((file) => file.getExtractedFilePath().toLowerCase().endsWith('.gdi'))
.map(async (gdiFile) => {
try {
const cueData = await util.promisify(fs.readFile)(gdiFile.getFilePath());
const { name: filePrefix } = path.parse(gdiFile.getFilePath());
const gdiContents = `${cueData
.toString()
.split(/\r?\n/)
.filter((line) => line.length > 0)
// Replace the chdman-generated track files with TOSEC-style track filenames
.map((line) => line.replace(filePrefix, 'track').replaceAll('"', ''))
.join('\r\n')}\r\n`;
const trackFilePaths = gdiContents
.trim()
.split(/\r?\n/)
.slice(1)
.map((line) => line.split(' ')[4]);
const trackFiles = trackFilePaths
.map((trackFilePath) => path.join(path.dirname(gdiFile.getFilePath()), trackFilePath))
.map((trackFilePath) => rawFilePathsToFiles.get(trackFilePath))
.filter((file) => file !== undefined);
if (trackFiles.length === 0) {
return undefined;
}
const gameName = DATGameInferrer.getGameName(gdiFile);
return [gameName, [gdiFile, ...trackFiles]];
}
catch {
return undefined;
}
}))).filter((result) => result !== undefined);
this.progressBar.logTrace(`inferred ${results.length.toLocaleString()} games from cue files`);
return results;
}
inferRawFiles(romFiles) {
this.progressBar.logTrace(`inferring games from raw files from ${romFiles.length.toLocaleString()} file${romFiles.length === 1 ? '' : 's'}`);
const results = romFiles
.filter((file) => !(file instanceof ArchiveEntry))
.reduce((map, file) => {
const gameName = DATGameInferrer.getGameName(file);
if (map.has(gameName)) {
map.get(gameName)?.push(file);
}
else {
map.set(gameName, [file]);
}
return map;
}, new Map());
this.progressBar.logTrace(`inferred ${results.size.toLocaleString()} games from raw files`);
return [...results.entries()];
}
}