UNPKG

igir

Version:

🕹 A zero-setup ROM collection manager that sorts, filters, extracts or archives, patches, and reports on collections of any size on any OS.

621 lines (620 loc) • 24.9 kB
import os from "node:os"; import path from "node:path"; import async from "async"; import chalk from "chalk"; import isAdmin from "is-admin"; import { CHDType } from "../packages/chdman/index.js"; import CandidateWriterSemaphore from "./async/candidateWriterSemaphore.js"; import FileMoveMutex from "./async/fileMoveMutex.js"; import MappableSemaphore from "./async/mappableSemaphore.js"; import Timer from "./async/timer.js"; import FileCache from "./cache/fileCache.js"; import { logger } from "./console/logger.js"; import MultiBar from "./console/multiBar.js"; import { ProgressBarSymbol } from "./console/progressBar.js"; import IgirException from "./exceptions/igirException.js"; import FileFactory from "./factories/fileFactory.js"; import Package from "./globals/package.js"; import Temp from "./globals/temp.js"; import ArchiveEntry from "./models/files/archives/archiveEntry.js"; import Chd from "./models/files/archives/chd/chd.js"; import File from "./models/files/file.js"; import { ChecksumBitmask, ChecksumBitmaskInverted } from "./models/files/fileChecksums.js"; import Options, { InputChecksumArchivesMode, LinkMode } from "./models/options.js"; import CandidateArchiveFileHasher from "./modules/candidates/candidateArchiveFileHasher.js"; import CandidateCombiner from "./modules/candidates/candidateCombiner.js"; import CandidateExtensionCorrector from "./modules/candidates/candidateExtensionCorrector.js"; import CandidateGenerator from "./modules/candidates/candidateGenerator.js"; import CandidateMergeSplitValidator from "./modules/candidates/candidateMergeSplitValidator.js"; import CandidatePatchGenerator from "./modules/candidates/candidatePatchGenerator.js"; import CandidatePostProcessor from "./modules/candidates/candidatePostProcessor.js"; import CandidateValidator from "./modules/candidates/candidateValidator.js"; import CandidateWriter from "./modules/candidates/candidateWriter.js"; import OutputFactory from "./modules/candidates/utils/outputFactory.js"; import DirectoryCleaner from "./modules/cleaners/directoryCleaner.js"; import InputSubdirectoriesDeleter from "./modules/cleaners/inputSubdirectoriesDeleter.js"; import MovedROMDeleter from "./modules/cleaners/movedRomDeleter.js"; import DATCombiner from "./modules/dats/datCombiner.js"; import DATDiscMerger from "./modules/dats/datDiscMerger.js"; import DATFilter from "./modules/dats/datFilter.js"; import DATGameInferrer from "./modules/dats/datGameInferrer.js"; import DATMergerSplitter from "./modules/dats/datMergerSplitter.js"; import DATParentInferrer from "./modules/dats/datParentInferrer.js"; import DATPreferer from "./modules/dats/datPreferer.js"; import DATScanner from "./modules/dats/datScanner.js"; import Dir2DatCreator from "./modules/dir2DatCreator.js"; import FixdatCreator from "./modules/fixdatCreator.js"; import PatchScanner from "./modules/patchScanner.js"; import PlaylistCreator from "./modules/playlistCreator.js"; import ReportGenerator from "./modules/reportGenerator.js"; import ROMHeaderProcessor from "./modules/roms/romHeaderProcessor.js"; import ROMIndexer from "./modules/roms/romIndexer.js"; import ROMScanner from "./modules/roms/romScanner.js"; import ROMTrimProcessor from "./modules/roms/romTrimProcessor.js"; import StatusGenerator from "./modules/statusGenerator.js"; import ArrayUtil from "./utils/arrayUtil.js"; import FsUtil from "./utils/fsUtil.js"; import IntlUtil from "./utils/intlUtil.js"; class Igir { options; multiBar; constructor(options) { this.options = options; this.multiBar = MultiBar.create(); } /** * The main method for this app. */ async main() { Temp.setTempDir(this.options.getTempDir()); if (this.options.shouldLink() && this.options.getLinkMode() === LinkMode.SYMLINK && process.platform === "win32") { logger.trace("checking Windows for symlink permissions"); if (!await FsUtil.canSymlink(Temp.getTempDir())) { if (!await isAdmin()) { throw new IgirException( `${Package.NAME} does not have permissions to create symlinks, please try running as administrator` ); } throw new IgirException(`${Package.NAME} does not have permissions to create symlinks`); } logger.trace("Windows has symlink permissions"); } if (this.options.shouldLink() && this.options.getLinkMode() === LinkMode.HARDLINK) { const outputDirRoot = this.options.getOutputDirRoot(); if (!await FsUtil.canHardlink(outputDirRoot)) { throw new IgirException( `the filesystem that '${outputDirRoot}' is on does not support hard-linking` ); } } const fileCache = new FileCache(); if (this.options.getDisableCache()) { logger.trace("disabling the file cache"); fileCache.disable(); } else { const cachePath = await this.getCachePath(); if (cachePath !== void 0 && process.env.NODE_ENV !== "test") { if (await FsUtil.exists(cachePath)) { logger.trace(`loading the existing file cache at '${cachePath}'`); } else { logger.trace(`creating a new file cache at '${cachePath}'`); } await fileCache.loadFile(cachePath); } else { logger.trace("not using a file for the file cache"); } } const fileFactory = new FileFactory(fileCache); const readerSemaphore = new MappableSemaphore(this.options.getReaderThreads()); const writerSemaphore = new CandidateWriterSemaphore(this.options.getWriterThreads()); const moveMutex = new FileMoveMutex(this.options.getReaderThreads() * 100); let dats = await this.processDATScanner(fileFactory, readerSemaphore); const indexedRoms = await this.processROMScanner( dats, fileFactory, readerSemaphore, this.determineScanningBitmask(dats), this.determineScanningChecksumArchives(dats) ); const roms = indexedRoms.getFiles(); const patches = await this.processPatchScanner(fileFactory, readerSemaphore); const datProcessProgressBar = this.multiBar.addSingleBar({ name: chalk.underline("Processing DATs"), symbol: ProgressBarSymbol.NONE, total: dats.length, progressBarSizeMultiplier: 2 }); if (dats.length === 0) { dats = await new DATGameInferrer(this.options, datProcessProgressBar).infer(roms); datProcessProgressBar.setTotal(dats.length); } if (dats.length <= 1) { datProcessProgressBar.delete(); } const candidateWriterResults = { wrote: [], moved: [] }; const filesToExcludeFromCleaning = []; let romOutputDirs = []; const datsStatuses = []; logger.trace( `processing ${IntlUtil.toLocaleString(dats.length)} DAT${dats.length === 1 ? "" : "s"}` ); await async.eachLimit(dats, this.options.getDatThreads(), async (dat) => { datProcessProgressBar.incrementInProgress(); const progressBar = this.multiBar.addSingleBar({ name: dat.getDisplayName(), symbol: ProgressBarSymbol.WAITING, total: dat.getParents().length }); const processedDat = this.processDAT(progressBar, dat); const candidates = await this.generateCandidates( progressBar, fileFactory, readerSemaphore, processedDat, indexedRoms, patches ); for (const candidate of candidates) { for (const romWithFiles of candidate.getRomsWithFiles()) { if (!romWithFiles.getInputFile().getCanBeCandidateInput()) { filesToExcludeFromCleaning.push(romWithFiles.getOutputFile()); } } } romOutputDirs = [...romOutputDirs, ...this.getCandidateOutputDirs(processedDat, candidates)]; const writerResults = await new CandidateWriter( this.options, progressBar, fileFactory, writerSemaphore, moveMutex ).write(processedDat, candidates); for (const moved of writerResults.moved) candidateWriterResults.moved.push(moved); for (const wrote of writerResults.wrote) candidateWriterResults.wrote.push(wrote); const playlistPaths = await new PlaylistCreator(this.options, progressBar).write( processedDat, candidates ); await Promise.all( playlistPaths.map(async (filePath) => { filesToExcludeFromCleaning.push(await File.fileOf({ filePath })); }) ); const dir2DatPath = await new Dir2DatCreator(this.options, progressBar).create( processedDat, candidates ); if (dir2DatPath) { filesToExcludeFromCleaning.push(await File.fileOf({ filePath: dir2DatPath })); } const fixdatPath = await new FixdatCreator(this.options, progressBar).create( processedDat, candidates ); if (fixdatPath) { filesToExcludeFromCleaning.push(await File.fileOf({ filePath: fixdatPath })); } const datStatus = new StatusGenerator(this.options, progressBar).generate( processedDat, candidates ); datsStatuses.push(datStatus); progressBar.finish( [ datStatus.toConsole(this.options), dir2DatPath ? `dir2dat: ${dir2DatPath}` : void 0, fixdatPath ? `Fixdat: ${fixdatPath}` : void 0 ].filter((line) => line !== void 0 && line.length > 0).join("\n") ); if (candidates.length > 0 || this.options.shouldDir2Dat() || this.options.shouldFixdat()) { progressBar.freeze(); } else { progressBar.delete(); } logger.trace("done processing DAT"); datProcessProgressBar.incrementCompleted(); }); logger.trace( `done processing ${IntlUtil.toLocaleString(dats.length)} DAT${dats.length === 1 ? "" : "s"}` ); datProcessProgressBar.finishWithItems(dats.length, "DAT", "processed"); datProcessProgressBar.delete(); const writtenOutputFiles = candidateWriterResults.wrote.flatMap( (wc) => wc.getRomsWithFiles().map((rwf) => rwf.getOutputFile()) ); await this.deleteMovedRoms(indexedRoms, candidateWriterResults.moved, writtenOutputFiles); const cleanedOutputFiles = await this.processOutputCleaner(romOutputDirs, [ // Do not clean output ROMs ...writtenOutputFiles, // Do not clean any other files written (dir2dats, fixdats, playlists, etc.) ...filesToExcludeFromCleaning ]); await this.processReportGenerator(roms, cleanedOutputFiles, datsStatuses); Timer.cancelAll(); } async getCachePath() { const defaultFileName = `${Package.NAME}.cache`; let cachePath = this.options.getCachePath(); if (cachePath !== void 0 && await FsUtil.isDirectory(cachePath)) { cachePath = path.join(cachePath, defaultFileName); logger.warn( `A directory was provided for the cache path instead of a file, using '${cachePath}' instead` ); } if (cachePath !== void 0) { if (await FsUtil.isWritable(cachePath)) { return cachePath; } logger.warn("Provided cache path isn't writable, using the default path"); } const cachePathCandidates = [ path.join(os.homedir(), defaultFileName), path.join(process.cwd(), defaultFileName) ].filter((filePath) => filePath.length > 0 && !filePath.startsWith(os.tmpdir())).reduce(ArrayUtil.reduceUnique(), []); const exists = await Promise.all( cachePathCandidates.map(async (pathCandidate) => await FsUtil.exists(pathCandidate)) ); const existsCachePath = cachePathCandidates.find((_, idx) => exists[idx]); if (existsCachePath !== void 0) { return existsCachePath; } const writable = await Promise.all( cachePathCandidates.map(async (pathCandidate) => await FsUtil.isWritable(pathCandidate)) ); const writableCachePath = cachePathCandidates.find((_, idx) => writable[idx]); if (writableCachePath !== void 0) { return writableCachePath; } return void 0; } async processDATScanner(fileFactory, readableSemaphore) { if (this.options.shouldDir2Dat()) { return []; } if (!this.options.usingDats()) { logger.warn("No DAT files provided, consider using some for the best results!"); return []; } const progressBar = this.multiBar.addSingleBar({ name: "Scanning for DATs" }); let dats = await new DATScanner( this.options, progressBar, fileFactory, readableSemaphore ).scan(); if (dats.length === 0) { throw new IgirException("No valid DAT files found!"); } if (dats.length === 1) { for (const [isOptionEnabled, option] of [ [this.options.getDirDatName(), "--dir-dat-name"], [this.options.getDirDatDescription(), "--dir-dat-description"] ]) { if (!isOptionEnabled) { continue; } logger.warn( `${option} is most helpful when processing multiple DATs, only one DAT was found` ); } } if (this.options.getDatCombine()) { progressBar.resetProgress(1); dats = [new DATCombiner(progressBar).combine(dats)]; } progressBar.finishWithItems( dats.length, "DAT", this.options.getDatCombine() ? "combined" : "found" ); progressBar.freeze(); return dats; } determineScanningBitmask(dats) { const minimumChecksum = this.options.getInputChecksumMin() ?? ChecksumBitmask.NONE; const maximumChecksum = this.options.getInputChecksumMax() ?? Object.values(ChecksumBitmask).at(-1) ?? minimumChecksum; let matchChecksum = minimumChecksum; if (this.options.getPatchFileCount() > 0 && !(matchChecksum & ChecksumBitmask.CRC32)) { matchChecksum |= ChecksumBitmask.CRC32; logger.trace("using patch files, enabling CRC32 file checksums"); } if (this.options.shouldDir2Dat()) { for (const bitmask of Object.values(ChecksumBitmask)) { if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled (matchChecksum & bitmask) !== 0) { continue; } matchChecksum |= bitmask; logger.trace( `generating a dir2dat, enabling ${ChecksumBitmaskInverted[bitmask]} file checksums` ); } } if (dats.length === 0) { for (const bitmask of Object.values(ChecksumBitmask)) { if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled (matchChecksum & bitmask) !== 0) { continue; } matchChecksum |= bitmask; logger.trace( `no DATs provided, enabling ${ChecksumBitmaskInverted[bitmask]} file checksums` ); } } for (const dat of dats) { const datMinimumRomBitmask = dat.getRequiredRomChecksumBitmask(); for (const bitmask of Object.values(ChecksumBitmask)) { if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled (matchChecksum & bitmask) !== 0 || // Should not be enabled for this DAT (datMinimumRomBitmask & bitmask) === 0) { continue; } matchChecksum |= bitmask; logger.trace( `${dat.getName()}: needs ${ChecksumBitmaskInverted[bitmask]} file checksums for ROMs, enabling` ); } if (this.options.getExcludeDisks()) { continue; } const datMinimumDiskBitmask = dat.getRequiredDiskChecksumBitmask(); for (const bitmask of Object.values(ChecksumBitmask)) { if (bitmask < minimumChecksum || bitmask > maximumChecksum || // Has already been enabled (matchChecksum & bitmask) !== 0 || // Should not be enabled for this DAT (datMinimumDiskBitmask & bitmask) === 0) { continue; } matchChecksum |= bitmask; logger.trace( `${dat.getName()}: needs ${ChecksumBitmaskInverted[bitmask]} file checksums for disks, enabling` ); } } if (matchChecksum === ChecksumBitmask.NONE) { matchChecksum |= ChecksumBitmask.CRC32; logger.trace("at least one checksum algorithm is required, enabling CRC32 file checksums"); } return matchChecksum; } determineScanningChecksumArchives(dats) { if (this.options.getInputChecksumArchives() === InputChecksumArchivesMode.NEVER) { return false; } if (this.options.getInputChecksumArchives() === InputChecksumArchivesMode.ALWAYS) { return true; } return dats.some( (dat) => dat.isMame() && dat.getGames().some( (game) => game.getRoms().some((rom) => { const isArchive = FileFactory.isExtensionArchive(rom.getName()); if (isArchive) { logger.trace( `${dat.getName()}: contains archives, enabling checksum calculation of raw archive contents` ); return true; } return false; }) ) ); } async processROMScanner(dats, fileFactory, readerSemaphore, checksumBitmask, shouldChecksumArchives) { const romProgressBar = this.multiBar.addSingleBar({ name: "Scanning for ROMs" }); const rawRomFiles = await new ROMScanner( this.options, romProgressBar, fileFactory, readerSemaphore ).scan(checksumBitmask, shouldChecksumArchives); if (dats.some((dat) => !dat.isMame())) { const chds = rawRomFiles.filter((file) => file instanceof ArchiveEntry).map((file) => file.getArchive()).filter((file) => file instanceof Chd); const chdToType = await Promise.all( chds.map(async (chd) => [chd, (await chd.getInfo()).type]) ); const cdRom = chdToType.find(([, type]) => type === CHDType.CD_ROM); if (cdRom !== void 0) { logger.warn( `${cdRom[0].getFilePath()}: quick checksumming will not process .cue/.bin files in CD-ROM CHDs!` ); } const gdRom = chdToType.find(([, type]) => type === CHDType.GD_ROM); if (gdRom !== void 0) { logger.warn( `${gdRom[0].getFilePath()}: quick checksumming will not process .gdi/.bin/.raw files in GD-ROM CHDs!` ); } } const romScannerProgressBarName = romProgressBar.getName(); romProgressBar.setName("Detecting ROM headers"); const romFilesWithHeaders = await new ROMHeaderProcessor( this.options, romProgressBar, fileFactory, readerSemaphore ).process(rawRomFiles); romProgressBar.setName("Detecting ROM trimming"); const romFilesWithTrimming = await new ROMTrimProcessor( this.options, romProgressBar, fileFactory, readerSemaphore ).process(romFilesWithHeaders); romProgressBar.setName("Indexing ROMs"); const indexedRomFiles = new ROMIndexer(this.options, romProgressBar).index( romFilesWithTrimming ); romProgressBar.setName(romScannerProgressBarName ?? ""); romProgressBar.finishWithItems(romFilesWithTrimming.length, "file", "found"); romProgressBar.freeze(); return indexedRomFiles; } async processPatchScanner(fileFactory, readerSemaphore) { if (!this.options.getPatchFileCount()) { return []; } const progressBar = this.multiBar.addSingleBar({ name: "Scanning for patches" }); const patches = await new PatchScanner( this.options, progressBar, fileFactory, readerSemaphore ).scan(); progressBar.finishWithItems(patches.length, "patch", "found"); progressBar.freeze(); return patches; } processDAT(progressBar, dat) { return [ (dat2) => new DATParentInferrer(this.options, progressBar).infer(dat2), (dat2) => new DATMergerSplitter(this.options, progressBar).merge(dat2), (dat2) => new DATDiscMerger(this.options, progressBar).merge(dat2), (dat2) => new DATFilter(this.options, progressBar).filter(dat2), (dat2) => new DATPreferer(this.options, progressBar).prefer(dat2) ].reduce((processedDat, processor) => { return processor(processedDat); }, dat); } async generateCandidates(progressBar, fileFactory, readerSemaphore, dat, indexedRoms, patches) { return await [ // Generate the initial set of candidates async () => await new CandidateGenerator( this.options, progressBar, fileFactory, readerSemaphore ).generate(dat, indexedRoms), // Add patched candidates (candidates) => new CandidatePatchGenerator(this.options, progressBar).generate(dat, candidates, patches), // Correct output filename extensions async (candidates) => await new CandidateExtensionCorrector( this.options, progressBar, fileFactory, readerSemaphore ).correct(dat, candidates), /** * Delay calculating checksums for {@link ArchiveFile}s until after the above steps for * efficiency */ async (candidates) => await new CandidateArchiveFileHasher( this.options, progressBar, fileFactory, readerSemaphore ).hash(dat, candidates), // Finalize output file paths (candidates) => new CandidatePostProcessor(this.options, progressBar).process(dat, candidates), // Validate candidates (candidates) => { const invalidCandidates = new CandidateValidator(this.options, progressBar).validate( dat, candidates ); if (invalidCandidates.length > 0) { return []; } return candidates; }, // Validate merge/split (candidates) => { new CandidateMergeSplitValidator(this.options, progressBar).validate(dat, candidates); return candidates; }, // Combine candidates into one (candidates) => new CandidateCombiner(this.options, progressBar).combine(dat, candidates) ].reduce( async (candidatesPromise, processor) => { const candidates = await candidatesPromise; return await processor(candidates); }, Promise.resolve([]) ); } /** * Find all ROM output paths for a DAT and its candidates. */ getCandidateOutputDirs(dat, candidates) { return candidates.flatMap( (candidate) => candidate.getRomsWithFiles().flatMap( (romWithFiles) => OutputFactory.getPath( // Parse the output directory, as supplied by the user, ONLY replacing tokens in the // path and NOT respecting any `--dir-*` options. new Options({ commands: [...this.options.getCommands()], output: this.options.getOutput(), outputConsoleTokens: this.options.getOutputConsoleTokens() }), dat, candidate.getGame(), romWithFiles.getRom(), romWithFiles.getInputFile() ).dir ) ).reduce(ArrayUtil.reduceUnique(), []); } async deleteMovedRoms(indexedRoms, movedWriteCandidates, writtenFilesToExclude) { if (movedWriteCandidates.length === 0) { return; } const progressBarName = "Deleting moved files"; const progressBar = this.multiBar.addSingleBar({ name: progressBarName }); const deletedFilePaths = await new MovedROMDeleter(this.options, progressBar).delete( indexedRoms, movedWriteCandidates, writtenFilesToExclude ); progressBar.setName("Deleting empty input subdirectories"); await new InputSubdirectoriesDeleter(this.options, progressBar).delete( movedWriteCandidates.flatMap((wc) => wc.getRomsWithFiles().map((rwf) => rwf.getInputFile())) ); progressBar.setName(progressBarName); progressBar.finishWithItems(deletedFilePaths.length, "moved file", "deleted"); if (deletedFilePaths.length > 0) { progressBar.freeze(); } else { progressBar.delete(); } } async processOutputCleaner(dirsToClean, writtenFilesToExclude) { if (!this.options.shouldWrite() || !this.options.shouldClean() || dirsToClean.length === 0) { return []; } const progressBar = this.multiBar.addSingleBar({ name: "Cleaning output directory" }); const uniqueDirsToClean = dirsToClean.reduce(ArrayUtil.reduceUnique(), []); const filesCleaned = await new DirectoryCleaner(this.options, progressBar).clean( uniqueDirsToClean, writtenFilesToExclude ); progressBar.finishWithItems(filesCleaned.length, "file", "recycled"); progressBar.freeze(); return filesCleaned; } async processReportGenerator(scannedRomFiles, cleanedOutputFiles, datsStatuses) { if (!this.options.shouldReport()) { return; } const reportProgressBar = this.multiBar.addSingleBar({ name: "Generating report", symbol: ProgressBarSymbol.WRITING }); await new ReportGenerator(this.options, reportProgressBar).generate( scannedRomFiles, cleanedOutputFiles, datsStatuses ); } } export { Igir as default }; //# sourceMappingURL=igir.js.map