igir
Version:
🕹 A zero-setup ROM collection manager that sorts, filters, extracts or archives, patches, and reports on collections of any size on any OS.
435 lines (434 loc) • 16 kB
JavaScript
import child_process from "node:child_process";
import path from "node:path";
import { parse } from "@fast-csv/parse";
import async from "async";
import { ProgressBarSymbol } from "../../console/progressBar.js";
import IgirException from "../../exceptions/igirException.js";
import Defaults from "../../globals/defaults.js";
import DATObject from "../../models/dats/datObject.js";
import Disk from "../../models/dats/disk.js";
import Game from "../../models/dats/game.js";
import Header from "../../models/dats/logiqx/header.js";
import LogiqxDAT from "../../models/dats/logiqx/logiqxDat.js";
import MameDAT from "../../models/dats/mame/mameDat.js";
import ROM from "../../models/dats/rom.js";
import SoftwareListDAT from "../../models/dats/softwarelist/softwareListDat.js";
import SoftwareListsDAT from "../../models/dats/softwarelist/softwareListsDat.js";
import ArchiveEntry from "../../models/files/archives/archiveEntry.js";
import { ChecksumBitmask } from "../../models/files/fileChecksums.js";
import ArrayUtil from "../../utils/arrayUtil.js";
import BufferUtil from "../../utils/bufferUtil.js";
import FsUtil from "../../utils/fsUtil.js";
import IntlUtil from "../../utils/intlUtil.js";
import Scanner from "../scanner.js";
import CMProParser from "./parsers/cmProParser.js";
import GameGrouper from "./utils/gameGrouper.js";
class DATScanner extends Scanner {
constructor(options, progressBar, fileFactory, mappableSemaphore) {
super(options, progressBar, fileFactory, mappableSemaphore, DATScanner.name);
}
/**
* Scan files and parse {@link DAT}s.
*/
async scan() {
this.prefixedLogger.trace("scanning DAT files");
this.progressBar.setSymbol(ProgressBarSymbol.FILE_SCANNING);
this.progressBar.resetProgress(0);
const datFilePaths = await this.options.scanDatFilesWithoutExclusions((increment) => {
this.progressBar.incrementTotal(increment);
});
if (datFilePaths.length === 0) {
return [];
}
this.prefixedLogger.trace(
`found ${IntlUtil.toLocaleString(datFilePaths.length)} DAT file${datFilePaths.length === 1 ? "" : "s"}`
);
this.progressBar.resetProgress(datFilePaths.length);
this.prefixedLogger.trace("enumerating DAT archives");
const datFiles = await this.getUniqueFilesFromPaths(datFilePaths, ChecksumBitmask.CRC32);
this.progressBar.resetProgress(datFiles.length);
const downloadedDats = await this.downloadDats(datFiles);
this.progressBar.resetProgress(downloadedDats.length);
const parsedDats = await this.parseDatFiles(downloadedDats);
this.prefixedLogger.trace("done scanning DAT files");
return parsedDats;
}
async downloadDats(datFiles) {
const datUrlFiles = datFiles.filter((datFile) => datFile.isURL());
if (datUrlFiles.length === 0) {
return datFiles;
}
this.prefixedLogger.trace(
`downloading ${IntlUtil.toLocaleString(datUrlFiles.length)} DAT${datUrlFiles.length === 1 ? "" : "s"} from URL${datUrlFiles.length === 1 ? "" : "s"}`
);
this.progressBar.setName("Downloading DATs");
this.progressBar.setSymbol(ProgressBarSymbol.DAT_DOWNLOADING);
return (await async.mapLimit(datFiles, Defaults.MAX_FS_THREADS, async (datFile) => {
try {
this.prefixedLogger.trace(`${datFile.toString()}: downloading`);
const downloadedDatFile = await datFile.downloadToTempPath();
this.prefixedLogger.trace(
`${datFile.toString()}: downloaded to '${downloadedDatFile.toString()}'`
);
return await this.getFilesFromPaths(
[downloadedDatFile.getFilePath()],
ChecksumBitmask.NONE
);
} catch (error) {
throw new IgirException(`failed to download '${datFile.toString()}': ${error}`);
}
})).flat();
}
// Parse each file into a DAT
async parseDatFiles(datFiles) {
this.prefixedLogger.trace(
`parsing ${IntlUtil.toLocaleString(datFiles.length)} DAT file${datFiles.length === 1 ? "" : "s"}`
);
if (datFiles.length === 0) {
return [];
}
this.progressBar.setName("Parsing DATs");
this.progressBar.setSymbol(ProgressBarSymbol.DAT_PARSING);
return (await this.mappableSemaphore.map(datFiles, async (datFile) => {
this.progressBar.incrementInProgress();
const childBar = this.progressBar.addChildBar({
name: datFile.toString(),
total: datFile.getSize(),
progressFormatter: FsUtil.sizeReadable.bind(FsUtil)
});
let dat;
try {
dat = await this.parseDatFile(datFile);
} catch (error) {
this.prefixedLogger.warn(`${datFile.toString()}: failed to parse DAT file: ${error}`);
} finally {
childBar.delete();
}
this.progressBar.incrementCompleted();
if (dat && this.shouldFilterOut(dat)) {
return void 0;
}
return dat;
})).filter((dat) => dat !== void 0).toSorted((a, b) => a.getName().localeCompare(b.getName()));
}
async parseDatFile(datFile) {
let dat;
if (!dat && !(datFile instanceof ArchiveEntry) && await FsUtil.isExecutable(datFile.getFilePath())) {
dat = await this.parseMameListxml(datFile);
}
dat ??= await datFile.createReadStream(async (readable) => {
const fileContents = await BufferUtil.fromReadable(readable);
return await this.parseDatContents(datFile, fileContents);
});
if (!dat) {
return dat;
}
if (dat.getGames().length === 1 && dat.getGames()[0].getIsBios() && dat.getGames()[0].getRoms().length > 10) {
const game = dat.getGames()[0];
dat = dat.withGames(
dat.getGames()[0].getRoms().filter(ArrayUtil.filterUniqueMapped((rom) => `${rom.getName()}|${rom.hashCode()}`)).map((rom) => {
const { dir, name } = path.parse(rom.getName());
const gameName = path.format({
dir,
name
});
return game.withProps({
name: gameName,
roms: [rom]
});
})
);
}
const size = dat.getGames().flatMap((game) => game.getRoms()).reduce((sum, rom) => sum + rom.getSize(), 0);
this.prefixedLogger.trace(
`${datFile.toString()}: ${FsUtil.sizeReadable(size)} of ${IntlUtil.toLocaleString(dat.getGames().length)} game${dat.getGames().length === 1 ? "" : "s"}, ${IntlUtil.toLocaleString(dat.getParents().length)} parent${dat.getParents().length === 1 ? "" : "s"} parsed`
);
return dat;
}
async parseMameListxml(mameExecutable) {
this.prefixedLogger.trace(
`${mameExecutable.toString()}: attempting to get ListXML from MAME executable`
);
let fileContents;
try {
fileContents = await new Promise((resolve, reject) => {
const proc = child_process.spawn(mameExecutable.getFilePath(), ["-listxml"], {
windowsHide: true
});
let output = "";
proc.stdout.on("data", (chunk) => {
output += chunk.toString();
});
proc.stderr.on("data", (chunk) => {
output += chunk.toString();
});
proc.on("close", (code) => {
if (code !== null && code > 0) {
reject(new Error(`exit code ${code}`));
return;
}
resolve(output);
});
proc.on("error", reject);
});
} catch (error) {
this.prefixedLogger.trace(
`${mameExecutable.toString()}: failed to get ListXML from MAME executable: ${error}`
);
return void 0;
}
return await this.parseDatContents(mameExecutable, fileContents);
}
async parseDatContents(datFile, fileContents) {
if (fileContents.length === 0) {
this.prefixedLogger.trace(`${datFile.toString()}: file is empty`);
return void 0;
}
const xmlDat = this.parseXmlDat(datFile, fileContents);
if (xmlDat) {
return xmlDat;
}
const cmproDatParsed = this.parseCmproDat(datFile, fileContents);
if (cmproDatParsed) {
return cmproDatParsed;
}
const smdbParsed = await this.parseSourceMaterialDatabase(datFile, fileContents);
if (smdbParsed) {
return smdbParsed;
}
this.prefixedLogger.trace(`${datFile.toString()}: failed to parse DAT file`);
return void 0;
}
parseXmlDat(datFile, fileContents) {
this.prefixedLogger.trace(
`${datFile.toString()}: attempting to parse ${FsUtil.sizeReadable(fileContents.length)} of XML`
);
let datObject;
try {
datObject = DATObject.fromXmlString(fileContents);
} catch (error) {
const message = error.message.replaceAll("\n", ", ");
this.prefixedLogger.trace(`${datFile.toString()}: failed to parse DAT XML: ${message}`);
return void 0;
}
this.prefixedLogger.trace(`${datFile.toString()}: parsed XML, deserializing to DAT`);
if (datObject.datafile) {
try {
return LogiqxDAT.fromObject(datObject.datafile, { filePath: datFile.getFilePath() });
} catch (error) {
this.prefixedLogger.trace(`${datFile.toString()}: failed to parse DAT object: ${error}`);
return void 0;
}
}
if (datObject.mame) {
try {
return MameDAT.fromObject(datObject.mame, { filePath: datFile.getFilePath() });
} catch (error) {
this.prefixedLogger.trace(
`${datFile.toString()}: failed to parse MAME DAT object: ${error}`
);
return void 0;
}
}
if (datObject.softwarelists) {
try {
return SoftwareListsDAT.fromObject(datObject.softwarelists, {
filePath: datFile.getFilePath()
});
} catch (error) {
this.prefixedLogger.trace(
`${datFile.toString()}: failed to parse software list DAT object: ${error}`
);
return void 0;
}
}
if (datObject.softwarelist) {
try {
return SoftwareListDAT.fromObject(datObject.softwarelist);
} catch (error) {
this.prefixedLogger.trace(
`${datFile.toString()}: failed to parse software list DAT object: ${error}`
);
return void 0;
}
}
this.prefixedLogger.trace(
`${datFile.toString()}: parsed XML, but failed to find a known DAT root`
);
return void 0;
}
parseCmproDat(datFile, fileContents) {
const fileContentsString = typeof fileContents === "string" ? fileContents : fileContents.toString();
if (!/^(clrmamepro|game|resource) \(\r?\n(\s.+\r?\n)+\)$/m.test(fileContentsString)) {
return void 0;
}
this.prefixedLogger.trace(`${datFile.toString()}: attempting to parse CMPro DAT`);
let cmproDat;
try {
cmproDat = new CMProParser(fileContentsString).parse();
} catch (error) {
this.prefixedLogger.trace(`${datFile.toString()}: failed to parse CMPro DAT: ${error}`);
return void 0;
}
this.prefixedLogger.trace(`${datFile.toString()}: parsed CMPro DAT, deserializing to DAT`);
const header = new Header({
name: cmproDat.clrmamepro?.name,
description: cmproDat.clrmamepro?.description,
version: cmproDat.clrmamepro?.version,
date: cmproDat.clrmamepro?.date,
author: cmproDat.clrmamepro?.author,
url: cmproDat.clrmamepro?.url,
comment: cmproDat.clrmamepro?.comment
});
let cmproDatGames = [];
if (cmproDat.game) {
if (Array.isArray(cmproDat.game)) {
cmproDatGames = cmproDat.game;
} else {
cmproDatGames = [cmproDat.game];
}
}
const games = cmproDatGames.flatMap((game) => {
const gameName = game.name ?? game.comment;
let gameRoms = [];
if (game.rom) {
if (Array.isArray(game.rom)) {
gameRoms = game.rom;
} else {
gameRoms = [game.rom];
}
}
const roms = gameRoms.map(
(entry) => new ROM({
name: entry.name ?? "",
size: Math.trunc(Number(entry.size ?? "0")),
crc32: entry.crc,
md5: entry.md5,
sha1: entry.sha1
})
);
let gameDisks = [];
if (game.disk) {
if (Array.isArray(game.disk)) {
gameDisks = game.disk;
} else {
gameDisks = [game.disk];
}
}
const disks = gameDisks.map(
(entry) => new Disk({
name: entry.name ?? "",
size: Math.trunc(Number(entry.size ?? "0")),
crc32: entry.crc,
md5: entry.md5,
sha1: entry.sha1
})
);
return new Game({
name: gameName,
categories: void 0,
description: game.description,
isBios: cmproDat.clrmamepro?.author?.toLowerCase() === "libretro" && cmproDat.clrmamepro.name?.toLowerCase() === "system" ? "yes" : "no",
isDevice: void 0,
cloneOf: game.cloneof,
romOf: game.romof,
genre: game.genre?.toString(),
release: void 0,
roms,
disks
});
});
return new LogiqxDAT({ filePath: datFile.getFilePath(), header, games });
}
/**
* @see https://github.com/frederic-mahe/Hardware-Target-Game-Database
*/
async parseSourceMaterialDatabase(datFile, fileContents) {
this.prefixedLogger.trace(`${datFile.toString()}: attempting to parse SMDB`);
let rows;
try {
rows = await DATScanner.parseSourceMaterialTsv(fileContents);
} catch (error) {
this.prefixedLogger.trace(`${datFile.toString()}: failed to parse SMDB: ${error}`);
return void 0;
}
if (rows.length === 0) {
this.prefixedLogger.trace(`${datFile.toString()}: failed to parse SMDB, file has no rows`);
return void 0;
}
this.prefixedLogger.trace(`${datFile.toString()}: parsed SMDB, deserializing to DAT`);
const rowNamesToRows = GameGrouper.groupMultiDiscGames(
rows,
(row) => row.name.replace(/\.[^.]*$/, "")
);
const games = Array.from(rowNamesToRows, ([gameName, rows2]) => {
const roms = rows2.map(
(row) => new ROM({
name: row.name,
size: Math.trunc(
Number(row.size !== void 0 && row.size.length > 0 ? row.size : "0")
),
crc32: row.crc,
md5: row.md5,
sha1: row.sha1,
sha256: row.sha256
})
);
return new Game({
name: gameName,
description: gameName,
roms
});
});
const datName = path.parse(datFile.getExtractedFilePath()).name;
return new LogiqxDAT({
filePath: datFile.getFilePath(),
header: new Header({
name: datName,
description: datName
}),
games
});
}
static async parseSourceMaterialTsv(fileContents) {
return await new Promise((resolve, reject) => {
const rows = [];
const stream = parse({
delimiter: " ",
quote: void 0,
headers: ["sha256", "name", "sha1", "md5", "crc", "size"]
}).validate(
(row) => row.name && (/^[0-9a-f]{8}$/.test(row.crc) || /^[0-9a-f]{32}$/.test(row.md5) || /^[0-9a-f]{40}$/.test(row.sha1) || /^[0-9a-f]{64}$/.test(row.sha256))
).on("error", reject).on("data", (row) => {
rows.push(row);
}).on("end", () => {
resolve(rows);
});
stream.write(fileContents);
stream.end();
});
}
shouldFilterOut(dat) {
const datNameRegex = this.options.getDatNameRegex();
if (datNameRegex?.every((regex) => !regex.test(dat.getName()))) {
return true;
}
const datNameRegexExclude = this.options.getDatNameRegexExclude();
if (datNameRegexExclude?.some((regex) => regex.test(dat.getName()))) {
return true;
}
const datDescription = dat.getDescription();
const datDescriptionRegex = this.options.getDatDescriptionRegex();
if (datDescription && datDescriptionRegex?.every((regex) => !regex.test(datDescription))) {
return true;
}
const datDescriptionRegexExclude = this.options.getDatDescriptionRegexExclude();
return Boolean(
datDescription && datDescriptionRegexExclude?.some((regex) => regex.test(datDescription))
);
}
}
export {
DATScanner as default
};
//# sourceMappingURL=datScanner.js.map