igir
Version:
🕹 A zero-setup ROM collection manager that sorts, filters, extracts or archives, patches, and reports on collections of any size on any OS.
621 lines (620 loc) • 24.9 kB
JavaScript
import os from "node:os";
import path from "node:path";
import async from "async";
import chalk from "chalk";
import isAdmin from "is-admin";
import { CHDType } from "../packages/chdman/index.js";
import CandidateWriterSemaphore from "./async/candidateWriterSemaphore.js";
import FileMoveMutex from "./async/fileMoveMutex.js";
import MappableSemaphore from "./async/mappableSemaphore.js";
import Timer from "./async/timer.js";
import FileCache from "./cache/fileCache.js";
import { logger } from "./console/logger.js";
import MultiBar from "./console/multiBar.js";
import { ProgressBarSymbol } from "./console/progressBar.js";
import IgirException from "./exceptions/igirException.js";
import FileFactory from "./factories/fileFactory.js";
import Package from "./globals/package.js";
import Temp from "./globals/temp.js";
import ArchiveEntry from "./models/files/archives/archiveEntry.js";
import Chd from "./models/files/archives/chd/chd.js";
import File from "./models/files/file.js";
import { ChecksumBitmask, ChecksumBitmaskInverted } from "./models/files/fileChecksums.js";
import Options, { InputChecksumArchivesMode, LinkMode } from "./models/options.js";
import CandidateArchiveFileHasher from "./modules/candidates/candidateArchiveFileHasher.js";
import CandidateCombiner from "./modules/candidates/candidateCombiner.js";
import CandidateExtensionCorrector from "./modules/candidates/candidateExtensionCorrector.js";
import CandidateGenerator from "./modules/candidates/candidateGenerator.js";
import CandidateMergeSplitValidator from "./modules/candidates/candidateMergeSplitValidator.js";
import CandidatePatchGenerator from "./modules/candidates/candidatePatchGenerator.js";
import CandidatePostProcessor from "./modules/candidates/candidatePostProcessor.js";
import CandidateValidator from "./modules/candidates/candidateValidator.js";
import CandidateWriter from "./modules/candidates/candidateWriter.js";
import OutputFactory from "./modules/candidates/utils/outputFactory.js";
import DirectoryCleaner from "./modules/cleaners/directoryCleaner.js";
import InputSubdirectoriesDeleter from "./modules/cleaners/inputSubdirectoriesDeleter.js";
import MovedROMDeleter from "./modules/cleaners/movedRomDeleter.js";
import DATCombiner from "./modules/dats/datCombiner.js";
import DATDiscMerger from "./modules/dats/datDiscMerger.js";
import DATFilter from "./modules/dats/datFilter.js";
import DATGameInferrer from "./modules/dats/datGameInferrer.js";
import DATMergerSplitter from "./modules/dats/datMergerSplitter.js";
import DATParentInferrer from "./modules/dats/datParentInferrer.js";
import DATPreferer from "./modules/dats/datPreferer.js";
import DATScanner from "./modules/dats/datScanner.js";
import Dir2DatCreator from "./modules/dir2DatCreator.js";
import FixdatCreator from "./modules/fixdatCreator.js";
import PatchScanner from "./modules/patchScanner.js";
import PlaylistCreator from "./modules/playlistCreator.js";
import ReportGenerator from "./modules/reportGenerator.js";
import ROMHeaderProcessor from "./modules/roms/romHeaderProcessor.js";
import ROMIndexer from "./modules/roms/romIndexer.js";
import ROMScanner from "./modules/roms/romScanner.js";
import ROMTrimProcessor from "./modules/roms/romTrimProcessor.js";
import StatusGenerator from "./modules/statusGenerator.js";
import ArrayUtil from "./utils/arrayUtil.js";
import FsUtil from "./utils/fsUtil.js";
import IntlUtil from "./utils/intlUtil.js";
class Igir {
options;
multiBar;
constructor(options) {
this.options = options;
this.multiBar = MultiBar.create();
}
/**
* The main method for this app.
*/
async main() {
Temp.setTempDir(this.options.getTempDir());
if (this.options.shouldLink() && this.options.getLinkMode() === LinkMode.SYMLINK && process.platform === "win32") {
logger.trace("checking Windows for symlink permissions");
if (!await FsUtil.canSymlink(Temp.getTempDir())) {
if (!await isAdmin()) {
throw new IgirException(
`${Package.NAME} does not have permissions to create symlinks, please try running as administrator`
);
}
throw new IgirException(`${Package.NAME} does not have permissions to create symlinks`);
}
logger.trace("Windows has symlink permissions");
}
if (this.options.shouldLink() && this.options.getLinkMode() === LinkMode.HARDLINK) {
const outputDirRoot = this.options.getOutputDirRoot();
if (!await FsUtil.canHardlink(outputDirRoot)) {
throw new IgirException(
`the filesystem that '${outputDirRoot}' is on does not support hard-linking`
);
}
}
const fileCache = new FileCache();
if (this.options.getDisableCache()) {
logger.trace("disabling the file cache");
fileCache.disable();
} else {
const cachePath = await this.getCachePath();
if (cachePath !== void 0 && process.env.NODE_ENV !== "test") {
if (await FsUtil.exists(cachePath)) {
logger.trace(`loading the existing file cache at '${cachePath}'`);
} else {
logger.trace(`creating a new file cache at '${cachePath}'`);
}
await fileCache.loadFile(cachePath);
} else {
logger.trace("not using a file for the file cache");
}
}
const fileFactory = new FileFactory(fileCache);
const readerSemaphore = new MappableSemaphore(this.options.getReaderThreads());
const writerSemaphore = new CandidateWriterSemaphore(this.options.getWriterThreads());
const moveMutex = new FileMoveMutex(this.options.getReaderThreads() * 100);
let dats = await this.processDATScanner(fileFactory, readerSemaphore);
const indexedRoms = await this.processROMScanner(
dats,
fileFactory,
readerSemaphore,
this.determineScanningBitmask(dats),
this.determineScanningChecksumArchives(dats)
);
const roms = indexedRoms.getFiles();
const patches = await this.processPatchScanner(fileFactory, readerSemaphore);
const datProcessProgressBar = this.multiBar.addSingleBar({
name: chalk.underline("Processing DATs"),
symbol: ProgressBarSymbol.NONE,
total: dats.length,
progressBarSizeMultiplier: 2
});
if (dats.length === 0) {
dats = await new DATGameInferrer(this.options, datProcessProgressBar).infer(roms);
datProcessProgressBar.setTotal(dats.length);
}
if (dats.length <= 1) {
datProcessProgressBar.delete();
}
const candidateWriterResults = {
wrote: [],
moved: []
};
const filesToExcludeFromCleaning = [];
let romOutputDirs = [];
const datsStatuses = [];
logger.trace(
`processing ${IntlUtil.toLocaleString(dats.length)} DAT${dats.length === 1 ? "" : "s"}`
);
await async.eachLimit(dats, this.options.getDatThreads(), async (dat) => {
datProcessProgressBar.incrementInProgress();
const progressBar = this.multiBar.addSingleBar({
name: dat.getDisplayName(),
symbol: ProgressBarSymbol.WAITING,
total: dat.getParents().length
});
const processedDat = this.processDAT(progressBar, dat);
const candidates = await this.generateCandidates(
progressBar,
fileFactory,
readerSemaphore,
processedDat,
indexedRoms,
patches
);
for (const candidate of candidates) {
for (const romWithFiles of candidate.getRomsWithFiles()) {
if (!romWithFiles.getInputFile().getCanBeCandidateInput()) {
filesToExcludeFromCleaning.push(romWithFiles.getOutputFile());
}
}
}
romOutputDirs = [...romOutputDirs, ...this.getCandidateOutputDirs(processedDat, candidates)];
const writerResults = await new CandidateWriter(
this.options,
progressBar,
fileFactory,
writerSemaphore,
moveMutex
).write(processedDat, candidates);
for (const moved of writerResults.moved) candidateWriterResults.moved.push(moved);
for (const wrote of writerResults.wrote) candidateWriterResults.wrote.push(wrote);
const playlistPaths = await new PlaylistCreator(this.options, progressBar).write(
processedDat,
candidates
);
await Promise.all(
playlistPaths.map(async (filePath) => {
filesToExcludeFromCleaning.push(await File.fileOf({ filePath }));
})
);
const dir2DatPath = await new Dir2DatCreator(this.options, progressBar).create(
processedDat,
candidates
);
if (dir2DatPath) {
filesToExcludeFromCleaning.push(await File.fileOf({ filePath: dir2DatPath }));
}
const fixdatPath = await new FixdatCreator(this.options, progressBar).create(
processedDat,
candidates
);
if (fixdatPath) {
filesToExcludeFromCleaning.push(await File.fileOf({ filePath: fixdatPath }));
}
const datStatus = new StatusGenerator(this.options, progressBar).generate(
processedDat,
candidates
);
datsStatuses.push(datStatus);
progressBar.finish(
[
datStatus.toConsole(this.options),
dir2DatPath ? `dir2dat: ${dir2DatPath}` : void 0,
fixdatPath ? `Fixdat: ${fixdatPath}` : void 0
].filter((line) => line !== void 0 && line.length > 0).join("\n")
);
if (candidates.length > 0 || this.options.shouldDir2Dat() || this.options.shouldFixdat()) {
progressBar.freeze();
} else {
progressBar.delete();
}
logger.trace("done processing DAT");
datProcessProgressBar.incrementCompleted();
});
logger.trace(
`done processing ${IntlUtil.toLocaleString(dats.length)} DAT${dats.length === 1 ? "" : "s"}`
);
datProcessProgressBar.finishWithItems(dats.length, "DAT", "processed");
datProcessProgressBar.delete();
const writtenOutputFiles = candidateWriterResults.wrote.flatMap(
(wc) => wc.getRomsWithFiles().map((rwf) => rwf.getOutputFile())
);
await this.deleteMovedRoms(indexedRoms, candidateWriterResults.moved, writtenOutputFiles);
const cleanedOutputFiles = await this.processOutputCleaner(romOutputDirs, [
// Do not clean output ROMs
...writtenOutputFiles,
// Do not clean any other files written (dir2dats, fixdats, playlists, etc.)
...filesToExcludeFromCleaning
]);
await this.processReportGenerator(roms, cleanedOutputFiles, datsStatuses);
Timer.cancelAll();
}
async getCachePath() {
const defaultFileName = `${Package.NAME}.cache`;
let cachePath = this.options.getCachePath();
if (cachePath !== void 0 && await FsUtil.isDirectory(cachePath)) {
cachePath = path.join(cachePath, defaultFileName);
logger.warn(
`A directory was provided for the cache path instead of a file, using '${cachePath}' instead`
);
}
if (cachePath !== void 0) {
if (await FsUtil.isWritable(cachePath)) {
return cachePath;
}
logger.warn("Provided cache path isn't writable, using the default path");
}
const cachePathCandidates = [
path.join(os.homedir(), defaultFileName),
path.join(process.cwd(), defaultFileName)
].filter((filePath) => filePath.length > 0 && !filePath.startsWith(os.tmpdir())).reduce(ArrayUtil.reduceUnique(), []);
const exists = await Promise.all(
cachePathCandidates.map(async (pathCandidate) => await FsUtil.exists(pathCandidate))
);
const existsCachePath = cachePathCandidates.find((_, idx) => exists[idx]);
if (existsCachePath !== void 0) {
return existsCachePath;
}
const writable = await Promise.all(
cachePathCandidates.map(async (pathCandidate) => await FsUtil.isWritable(pathCandidate))
);
const writableCachePath = cachePathCandidates.find((_, idx) => writable[idx]);
if (writableCachePath !== void 0) {
return writableCachePath;
}
return void 0;
}
async processDATScanner(fileFactory, readableSemaphore) {
if (this.options.shouldDir2Dat()) {
return [];
}
if (!this.options.usingDats()) {
logger.warn("No DAT files provided, consider using some for the best results!");
return [];
}
const progressBar = this.multiBar.addSingleBar({
name: "Scanning for DATs"
});
let dats = await new DATScanner(
this.options,
progressBar,
fileFactory,
readableSemaphore
).scan();
if (dats.length === 0) {
throw new IgirException("No valid DAT files found!");
}
if (dats.length === 1) {
for (const [isOptionEnabled, option] of [
[this.options.getDirDatName(), "--dir-dat-name"],
[this.options.getDirDatDescription(), "--dir-dat-description"]
]) {
if (!isOptionEnabled) {
continue;
}
logger.warn(
`${option} is most helpful when processing multiple DATs, only one DAT was found`
);
}
}
if (this.options.getDatCombine()) {
progressBar.resetProgress(1);
dats = [new DATCombiner(progressBar).combine(dats)];
}
progressBar.finishWithItems(
dats.length,
"DAT",
this.options.getDatCombine() ? "combined" : "found"
);
progressBar.freeze();
return dats;
}
determineScanningBitmask(dats) {
const minimumChecksum = this.options.getInputChecksumMin() ?? ChecksumBitmask.NONE;
const maximumChecksum = this.options.getInputChecksumMax() ?? Object.values(ChecksumBitmask).at(-1) ?? minimumChecksum;
let matchChecksum = minimumChecksum;
if (this.options.getPatchFileCount() > 0 && !(matchChecksum & ChecksumBitmask.CRC32)) {
matchChecksum |= ChecksumBitmask.CRC32;
logger.trace("using patch files, enabling CRC32 file checksums");
}
if (this.options.shouldDir2Dat()) {
for (const bitmask of Object.values(ChecksumBitmask)) {
if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled
(matchChecksum & bitmask) !== 0) {
continue;
}
matchChecksum |= bitmask;
logger.trace(
`generating a dir2dat, enabling ${ChecksumBitmaskInverted[bitmask]} file checksums`
);
}
}
if (dats.length === 0) {
for (const bitmask of Object.values(ChecksumBitmask)) {
if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled
(matchChecksum & bitmask) !== 0) {
continue;
}
matchChecksum |= bitmask;
logger.trace(
`no DATs provided, enabling ${ChecksumBitmaskInverted[bitmask]} file checksums`
);
}
}
for (const dat of dats) {
const datMinimumRomBitmask = dat.getRequiredRomChecksumBitmask();
for (const bitmask of Object.values(ChecksumBitmask)) {
if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled
(matchChecksum & bitmask) !== 0 || // Should not be enabled for this DAT
(datMinimumRomBitmask & bitmask) === 0) {
continue;
}
matchChecksum |= bitmask;
logger.trace(
`${dat.getName()}: needs ${ChecksumBitmaskInverted[bitmask]} file checksums for ROMs, enabling`
);
}
if (this.options.getExcludeDisks()) {
continue;
}
const datMinimumDiskBitmask = dat.getRequiredDiskChecksumBitmask();
for (const bitmask of Object.values(ChecksumBitmask)) {
if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled
(matchChecksum & bitmask) !== 0 || // Should not be enabled for this DAT
(datMinimumDiskBitmask & bitmask) === 0) {
continue;
}
matchChecksum |= bitmask;
logger.trace(
`${dat.getName()}: needs ${ChecksumBitmaskInverted[bitmask]} file checksums for disks, enabling`
);
}
}
if (matchChecksum === ChecksumBitmask.NONE) {
matchChecksum |= ChecksumBitmask.CRC32;
logger.trace("at least one checksum algorithm is required, enabling CRC32 file checksums");
}
return matchChecksum;
}
determineScanningChecksumArchives(dats) {
if (this.options.getInputChecksumArchives() === InputChecksumArchivesMode.NEVER) {
return false;
}
if (this.options.getInputChecksumArchives() === InputChecksumArchivesMode.ALWAYS) {
return true;
}
return dats.some(
(dat) => dat.isMame() && dat.getGames().some(
(game) => game.getRoms().some((rom) => {
const isArchive = FileFactory.isExtensionArchive(rom.getName());
if (isArchive) {
logger.trace(
`${dat.getName()}: contains archives, enabling checksum calculation of raw archive contents`
);
return true;
}
return false;
})
)
);
}
async processROMScanner(dats, fileFactory, readerSemaphore, checksumBitmask, shouldChecksumArchives) {
const romProgressBar = this.multiBar.addSingleBar({
name: "Scanning for ROMs"
});
const rawRomFiles = await new ROMScanner(
this.options,
romProgressBar,
fileFactory,
readerSemaphore
).scan(checksumBitmask, shouldChecksumArchives);
if (dats.some((dat) => !dat.isMame())) {
const chds = rawRomFiles.filter((file) => file instanceof ArchiveEntry).map((file) => file.getArchive()).filter((file) => file instanceof Chd);
const chdToType = await Promise.all(
chds.map(async (chd) => [chd, (await chd.getInfo()).type])
);
const cdRom = chdToType.find(([, type]) => type === CHDType.CD_ROM);
if (cdRom !== void 0) {
logger.warn(
`${cdRom[0].getFilePath()}: quick checksumming will not process .cue/.bin files in CD-ROM CHDs!`
);
}
const gdRom = chdToType.find(([, type]) => type === CHDType.GD_ROM);
if (gdRom !== void 0) {
logger.warn(
`${gdRom[0].getFilePath()}: quick checksumming will not process .gdi/.bin/.raw files in GD-ROM CHDs!`
);
}
}
const romScannerProgressBarName = romProgressBar.getName();
romProgressBar.setName("Detecting ROM headers");
const romFilesWithHeaders = await new ROMHeaderProcessor(
this.options,
romProgressBar,
fileFactory,
readerSemaphore
).process(rawRomFiles);
romProgressBar.setName("Detecting ROM trimming");
const romFilesWithTrimming = await new ROMTrimProcessor(
this.options,
romProgressBar,
fileFactory,
readerSemaphore
).process(romFilesWithHeaders);
romProgressBar.setName("Indexing ROMs");
const indexedRomFiles = new ROMIndexer(this.options, romProgressBar).index(
romFilesWithTrimming
);
romProgressBar.setName(romScannerProgressBarName ?? "");
romProgressBar.finishWithItems(romFilesWithTrimming.length, "file", "found");
romProgressBar.freeze();
return indexedRomFiles;
}
async processPatchScanner(fileFactory, readerSemaphore) {
if (!this.options.getPatchFileCount()) {
return [];
}
const progressBar = this.multiBar.addSingleBar({
name: "Scanning for patches"
});
const patches = await new PatchScanner(
this.options,
progressBar,
fileFactory,
readerSemaphore
).scan();
progressBar.finishWithItems(patches.length, "patch", "found");
progressBar.freeze();
return patches;
}
processDAT(progressBar, dat) {
return [
(dat2) => new DATParentInferrer(this.options, progressBar).infer(dat2),
(dat2) => new DATMergerSplitter(this.options, progressBar).merge(dat2),
(dat2) => new DATDiscMerger(this.options, progressBar).merge(dat2),
(dat2) => new DATFilter(this.options, progressBar).filter(dat2),
(dat2) => new DATPreferer(this.options, progressBar).prefer(dat2)
].reduce((processedDat, processor) => {
return processor(processedDat);
}, dat);
}
async generateCandidates(progressBar, fileFactory, readerSemaphore, dat, indexedRoms, patches) {
return await [
// Generate the initial set of candidates
async () => await new CandidateGenerator(
this.options,
progressBar,
fileFactory,
readerSemaphore
).generate(dat, indexedRoms),
// Add patched candidates
(candidates) => new CandidatePatchGenerator(this.options, progressBar).generate(dat, candidates, patches),
// Correct output filename extensions
async (candidates) => await new CandidateExtensionCorrector(
this.options,
progressBar,
fileFactory,
readerSemaphore
).correct(dat, candidates),
/**
* Delay calculating checksums for {@link ArchiveFile}s until after the above steps for
* efficiency
*/
async (candidates) => await new CandidateArchiveFileHasher(
this.options,
progressBar,
fileFactory,
readerSemaphore
).hash(dat, candidates),
// Finalize output file paths
(candidates) => new CandidatePostProcessor(this.options, progressBar).process(dat, candidates),
// Validate candidates
(candidates) => {
const invalidCandidates = new CandidateValidator(this.options, progressBar).validate(
dat,
candidates
);
if (invalidCandidates.length > 0) {
return [];
}
return candidates;
},
// Validate merge/split
(candidates) => {
new CandidateMergeSplitValidator(this.options, progressBar).validate(dat, candidates);
return candidates;
},
// Combine candidates into one
(candidates) => new CandidateCombiner(this.options, progressBar).combine(dat, candidates)
].reduce(
async (candidatesPromise, processor) => {
const candidates = await candidatesPromise;
return await processor(candidates);
},
Promise.resolve([])
);
}
/**
* Find all ROM output paths for a DAT and its candidates.
*/
getCandidateOutputDirs(dat, candidates) {
return candidates.flatMap(
(candidate) => candidate.getRomsWithFiles().flatMap(
(romWithFiles) => OutputFactory.getPath(
// Parse the output directory, as supplied by the user, ONLY replacing tokens in the
// path and NOT respecting any `--dir-*` options.
new Options({
commands: [...this.options.getCommands()],
output: this.options.getOutput(),
outputConsoleTokens: this.options.getOutputConsoleTokens()
}),
dat,
candidate.getGame(),
romWithFiles.getRom(),
romWithFiles.getInputFile()
).dir
)
).reduce(ArrayUtil.reduceUnique(), []);
}
async deleteMovedRoms(indexedRoms, movedWriteCandidates, writtenFilesToExclude) {
if (movedWriteCandidates.length === 0) {
return;
}
const progressBarName = "Deleting moved files";
const progressBar = this.multiBar.addSingleBar({ name: progressBarName });
const deletedFilePaths = await new MovedROMDeleter(this.options, progressBar).delete(
indexedRoms,
movedWriteCandidates,
writtenFilesToExclude
);
progressBar.setName("Deleting empty input subdirectories");
await new InputSubdirectoriesDeleter(this.options, progressBar).delete(
movedWriteCandidates.flatMap((wc) => wc.getRomsWithFiles().map((rwf) => rwf.getInputFile()))
);
progressBar.setName(progressBarName);
progressBar.finishWithItems(deletedFilePaths.length, "moved file", "deleted");
if (deletedFilePaths.length > 0) {
progressBar.freeze();
} else {
progressBar.delete();
}
}
async processOutputCleaner(dirsToClean, writtenFilesToExclude) {
if (!this.options.shouldWrite() || !this.options.shouldClean() || dirsToClean.length === 0) {
return [];
}
const progressBar = this.multiBar.addSingleBar({ name: "Cleaning output directory" });
const uniqueDirsToClean = dirsToClean.reduce(ArrayUtil.reduceUnique(), []);
const filesCleaned = await new DirectoryCleaner(this.options, progressBar).clean(
uniqueDirsToClean,
writtenFilesToExclude
);
progressBar.finishWithItems(filesCleaned.length, "file", "recycled");
progressBar.freeze();
return filesCleaned;
}
async processReportGenerator(scannedRomFiles, cleanedOutputFiles, datsStatuses) {
if (!this.options.shouldReport()) {
return;
}
const reportProgressBar = this.multiBar.addSingleBar({
name: "Generating report",
symbol: ProgressBarSymbol.WRITING
});
await new ReportGenerator(this.options, reportProgressBar).generate(
scannedRomFiles,
cleanedOutputFiles,
datsStatuses
);
}
}
export {
Igir as default
};
//# sourceMappingURL=igir.js.map