igir
Version:
🕹 A zero-setup ROM collection manager that sorts, filters, extracts or archives, patches, and reports on collections of any size on any OS.
407 lines (406 loc) • 15.7 kB
JavaScript
import { ValidationResultInverted } from "../../packages/torrentzip/index.js";
import { TZValidator } from "../../packages/torrentzip/index.js";
import { ValidationResult } from "../../packages/torrentzip/index.js";
import { ZipReader } from "../../packages/zip/index.js";
import { logger } from "../console/logger.js";
import ArchiveEntry from "../models/files/archives/archiveEntry.js";
import File from "../models/files/file.js";
import FileChecksums, { ChecksumBitmask } from "../models/files/fileChecksums.js";
import FileSignature from "../models/files/fileSignature.js";
import ROMHeader from "../models/files/romHeader.js";
import ROMPadding from "../models/files/romPadding.js";
import FsUtil from "../utils/fsUtil.js";
import Cache from "./cache.js";
const ValueType = {
FILE_CHECKSUMS: "F",
ARCHIVE_CHECKSUMS: "A",
ROM_HEADER: "H",
FILE_SIGNATURE: "S",
ROM_PADDING: `P${ROMPadding.getKnownFillBytesCount()}`,
TZ_VALIDATION: "Z"
};
class FileCache {
static VERSION = 6;
cache = new Cache();
enabled = true;
prefixedLogger = logger.child(FileCache.name);
/**
* Disable the cache, preventing it from being persisted to disk on {@link save}.
*/
disable() {
this.enabled = false;
}
/**
* Load the cache from a file on disk, then prune entries from older cache versions or stale
* value types so subsequent lookups only consider entries compatible with the current schema.
*/
async loadFile(cacheFilePath) {
this.prefixedLogger.trace(`loading: ${cacheFilePath}`);
this.cache = await new Cache({
filePath: cacheFilePath,
fileFlushMillis: 5 * 60 * 1e3,
saveOnExit: true
}).load();
await this.cache.delete(
new RegExp(
`^V(${[...Array.from({ length: FileCache.VERSION }).keys()].slice(1).join("|")})\\|`
)
);
await this.cache.delete(new RegExp(`\\|(?!(${Object.values(ValueType).join("|")}))[^|]+$`));
await this.cache.delete(/\|[HSZ]\d+$/);
this.prefixedLogger.trace(`loaded: ${cacheFilePath}`);
}
/**
* Persist the cache to its backing file, unless the cache has been disabled.
*/
async save() {
if (!this.enabled) {
return;
}
this.prefixedLogger.trace(`saving: ${this.cache.getFilePath()}`);
await this.cache.save();
this.prefixedLogger.trace(`saved: ${this.cache.getFilePath()}`);
}
async getOrComputeFileChecksums(filePath, checksumBitmask, callback, shouldForceRecompute = false) {
const stats = await FsUtil.stat(filePath);
const cacheKey = this.getCacheKey(filePath, void 0, ValueType.FILE_CHECKSUMS);
let computedFile;
const cachedValue = await this.cache.getOrCompute(
cacheKey,
async () => {
const checksums = await FileChecksums.hashFile(
filePath,
checksumBitmask,
void 0,
void 0,
callback
);
computedFile = await File.fileOf({ filePath, ...checksums }, checksumBitmask);
return {
fileSize: stats.size,
modifiedTimeSec: stats.mtimeS,
value: computedFile.toFileProps()
};
},
(cached) => {
if (shouldForceRecompute) {
return true;
}
if (cached.fileSize !== stats.size) {
this.prefixedLogger.trace(
`${filePath}: cache miss, cached file size ${cached.fileSize} !== real size ${stats.size}`
);
return true;
}
if (cached.modifiedTimeSec !== stats.mtimeS) {
this.prefixedLogger.trace(
`${filePath}: cache miss, cached file mtime ${cached.modifiedTimeSec} !== real mtime ${stats.mtimeS}`
);
return true;
}
const cachedFile2 = cached.value;
const existingBitmask = (cachedFile2.crc32 ? ChecksumBitmask.CRC32 : 0) | (cachedFile2.md5 ? ChecksumBitmask.MD5 : 0) | (cachedFile2.sha1 ? ChecksumBitmask.SHA1 : 0) | (cachedFile2.sha256 ? ChecksumBitmask.SHA256 : 0);
const remainingBitmask = checksumBitmask - (checksumBitmask & existingBitmask);
if (remainingBitmask > 0) {
this.prefixedLogger.trace(
`${filePath}: cache miss, cache is missing: ${FileChecksums.checksumBitmaskString(remainingBitmask)}`
);
return true;
}
return false;
}
);
if (computedFile) {
return computedFile;
}
const cachedFile = cachedValue.value;
return await File.fileOfObject(filePath, cachedFile);
}
async getOrComputeArchiveChecksums(archive, checksumBitmask, shouldForceRecompute = false, callback, shouldForceChecksumCalculation = false) {
const stats = await FsUtil.stat(archive.getFilePath());
if (stats.size === 0) {
return [];
}
const cacheKey = this.getCacheKey(
archive.getFilePath(),
archive.constructor.name,
ValueType.ARCHIVE_CHECKSUMS
);
let computedEntries;
const cachedValue = await this.cache.getOrCompute(
cacheKey,
async () => {
computedEntries = await archive.getArchiveEntries(
checksumBitmask,
callback,
shouldForceChecksumCalculation
);
return {
fileSize: stats.size,
modifiedTimeSec: stats.mtimeS,
value: computedEntries.map((entry) => entry.toEntryProps())
};
},
(cached) => {
if (shouldForceRecompute) {
return true;
}
if (cached.fileSize !== stats.size) {
this.prefixedLogger.trace(
`${archive.getFilePath()}: cache miss, cached file size ${cached.fileSize} !== real size ${stats.size}`
);
return true;
}
if (cached.modifiedTimeSec !== stats.mtimeS) {
this.prefixedLogger.trace(
`${archive.getFilePath()}: cache miss, cached file mtime ${cached.modifiedTimeSec} !== real mtime ${stats.mtimeS}`
);
return true;
}
const cachedEntries2 = cached.value;
if (cachedEntries2.length === 0) {
this.prefixedLogger.trace(
`${archive.getFilePath()}: cache miss, cache has zero archive entries`
);
return true;
}
const existingBitmask = (cachedEntries2.every((props) => props.crc32 !== void 0) ? ChecksumBitmask.CRC32 : 0) | (cachedEntries2.every((props) => props.md5 !== void 0) ? ChecksumBitmask.MD5 : 0) | (cachedEntries2.every((props) => props.sha1 !== void 0) ? ChecksumBitmask.SHA1 : 0) | (cachedEntries2.every((props) => props.sha256 !== void 0) ? ChecksumBitmask.SHA256 : 0);
const remainingBitmask = checksumBitmask - (checksumBitmask & existingBitmask);
if (remainingBitmask > 0) {
this.prefixedLogger.trace(
`${archive.getFilePath()}: cache miss, cache is missing: ${FileChecksums.checksumBitmaskString(remainingBitmask)}`
);
return true;
}
return false;
}
);
if (computedEntries) {
return computedEntries;
}
const cachedEntries = cachedValue.value;
return await Promise.all(
cachedEntries.map(async (props) => await ArchiveEntry.entryOfObject(archive, props))
);
}
async getOrComputeFileHeader(file) {
return await this.getOrComputeAny(
file,
ValueType.ROM_HEADER,
ROMHeader.getKnownHeaderCount(),
async () => {
const header = await file.createReadStream(
async (readable) => await ROMHeader.headerFromFileStream(readable)
);
return header?.getName();
},
(name) => ROMHeader.headerFromName(name)
);
}
async getOrComputeFileSignature(file, callback) {
return await this.getOrComputeAny(
file,
ValueType.FILE_SIGNATURE,
FileSignature.SIGNATURES.length,
async () => {
const signature = await file.createReadStream(
async (readable) => await FileSignature.signatureFromFileStream(readable, callback)
);
return signature?.getName();
},
(name) => FileSignature.signatureFromName(name)
);
}
/**
* Shared logic for caching file header and file signature lookups. Both cache a name string
* and reconstruct the object from that name on cache hit.
*/
async getOrComputeAny(file, valueType, currentVersion, runnable, fromName) {
if (file.getSize() === 0) {
return void 0;
}
const cacheKeys = this.getChecksumCacheKeys(file, valueType);
const isUsingFilePathKey = cacheKeys.length === 0;
if (isUsingFilePathKey) {
cacheKeys.push(this.getCacheKey(file.getFilePath(), void 0, valueType));
}
const stats = isUsingFilePathKey ? await FsUtil.stat(file.getFilePath()) : void 0;
const cachedValue = await this.cache.getOrComputeAnyKeys(
cacheKeys,
async () => {
const name = await runnable();
return {
fileSize: stats?.size,
modifiedTimeSec: stats?.mtimeS,
version: currentVersion,
value: name
};
},
(cached) => {
if (stats !== void 0 && cached.fileSize !== stats.size) {
this.prefixedLogger.trace(
`${file.getFilePath()}: cache miss, cached file size ${cached.fileSize} !== real size ${stats.size}`
);
return true;
}
if (stats !== void 0 && cached.modifiedTimeSec !== stats.mtimeS) {
this.prefixedLogger.trace(
`${file.getFilePath()}: cache miss, cached file mtime ${cached.modifiedTimeSec} !== real mtime ${stats.mtimeS}`
);
return true;
}
if (cached.value === void 0) {
if (cached.version !== currentVersion) {
this.prefixedLogger.trace(
`${file.getFilePath()}: cache miss, cached version ${cached.version} !== current version ${currentVersion}`
);
return true;
}
return false;
}
if (typeof cached.value !== "string" || fromName(cached.value) === void 0) {
this.prefixedLogger.trace(
// eslint-disable-next-line @typescript-eslint/no-base-to-string,@typescript-eslint/restrict-template-expressions
`${file.getFilePath()}: cache miss, cached value unrecognized: ${cached.value}`
);
return true;
}
return false;
}
);
const cachedName = cachedValue?.value;
if (cachedName === void 0) {
return void 0;
}
return fromName(cachedName);
}
async getOrComputeFilePaddings(file, callback) {
if (file.getSize() === 0) {
return [];
}
const cacheKeys = this.getChecksumCacheKeys(file, ValueType.ROM_PADDING);
if (cacheKeys.length === 0) {
cacheKeys.push(this.getCacheKey(file.getFilePath(), void 0, ValueType.ROM_PADDING));
}
const activeChecksums = Object.values(ChecksumBitmask).filter(
(bitmask) => file.getChecksumBitmask() & bitmask
);
const cachedResults = await this.cache.getOrComputeAllKeys(cacheKeys, async () => {
const paddings = await ROMPadding.paddingsFromFile(file, callback);
const paddingProps = paddings.map((padding) => padding.toROMPaddingProps());
const resultMap = /* @__PURE__ */ new Map();
for (const [i, bitmask] of activeChecksums.entries()) {
const perTypePaddings = paddingProps.map((props) => ({
paddedSize: props.paddedSize,
fillByte: props.fillByte,
...bitmask === ChecksumBitmask.CRC32 && props.crc32 !== void 0 && { crc32: props.crc32 },
...bitmask === ChecksumBitmask.MD5 && props.md5 !== void 0 && { md5: props.md5 },
...bitmask === ChecksumBitmask.SHA1 && props.sha1 !== void 0 && { sha1: props.sha1 },
...bitmask === ChecksumBitmask.SHA256 && props.sha256 !== void 0 && { sha256: props.sha256 }
}));
resultMap.set(cacheKeys[i], { value: perTypePaddings });
}
return resultMap;
});
const fillByteToRomPaddingProps = /* @__PURE__ */ new Map();
for (const cacheValue of cachedResults.values()) {
const paddingPropsList = cacheValue.value;
for (const element of paddingPropsList) {
const existing = fillByteToRomPaddingProps.get(element.fillByte) ?? {
paddedSize: element.paddedSize,
fillByte: element.fillByte
};
fillByteToRomPaddingProps.set(element.fillByte, {
...existing,
...element.crc32 !== void 0 && { crc32: element.crc32 },
...element.md5 !== void 0 && { md5: element.md5 },
...element.sha1 !== void 0 && { sha1: element.sha1 },
...element.sha256 !== void 0 && { sha256: element.sha256 }
});
}
}
return Array.from(fillByteToRomPaddingProps.values(), (props) => new ROMPadding(props));
}
async getOrComputeTzValidation(zip, shouldForceRecompute = false) {
if (!await FsUtil.exists(zip.getFilePath())) {
return ValidationResult.INVALID;
}
const stats = await FsUtil.stat(zip.getFilePath());
const currentVersion = Object.keys(ValidationResult).length;
const cacheKey = this.getCacheKey(zip.getFilePath(), void 0, ValueType.TZ_VALIDATION);
const cachedValue = await this.cache.getOrCompute(
cacheKey,
async () => {
let result;
try {
result = await TZValidator.validate(new ZipReader(zip.getFilePath()));
} catch {
result = ValidationResult.INVALID;
}
return {
fileSize: stats.size,
modifiedTimeSec: stats.mtimeS,
version: currentVersion,
value: ValidationResultInverted[result]
};
},
(cached) => {
if (shouldForceRecompute) {
return true;
}
if (cached.fileSize !== stats.size) {
this.prefixedLogger.trace(
`${zip.getFilePath()}: cache miss, cached file size ${cached.fileSize} !== real size ${stats.size}`
);
return true;
}
if (cached.modifiedTimeSec !== stats.mtimeS) {
this.prefixedLogger.trace(
`${zip.getFilePath()}: cache miss, cached file mtime ${cached.modifiedTimeSec} !== real mtime ${stats.mtimeS}`
);
return true;
}
const cachedResult2 = cached.value;
if (ValidationResult[cachedResult2] === void 0) {
this.prefixedLogger.trace(
`${zip.getFilePath()}: cache miss, cached value unrecognized: ${cachedResult2}`
);
return true;
}
if (ValidationResult[cachedResult2] === ValidationResult.INVALID) {
if (cached.version !== currentVersion) {
this.prefixedLogger.trace(
`${zip.getFilePath()}: cache miss, cached version ${cached.version} !== current version ${currentVersion}`
);
return true;
}
return false;
}
return false;
}
);
const cachedResult = cachedValue.value;
return ValidationResult[cachedResult];
}
getChecksumCacheKeys(file, valueType) {
const checksumEntries = [
[
ChecksumBitmask.CRC32,
file.getCrc32() === void 0 ? void 0 : `${file.getCrc32()}:${file.getSize()}`
],
[ChecksumBitmask.MD5, file.getMd5()],
[ChecksumBitmask.SHA1, file.getSha1()],
[ChecksumBitmask.SHA256, file.getSha256()]
];
return checksumEntries.filter(
(entry) => (file.getChecksumBitmask() & entry[0]) > 0 && entry[1] !== void 0
).map(([, checksum]) => this.getCacheKey("", checksum, valueType));
}
getCacheKey(filePath, fileSubIdentifier, valueType) {
return `V${FileCache.VERSION}|${filePath}|${fileSubIdentifier ?? ""}|${valueType}`;
}
}
export {
FileCache as default
};
//# sourceMappingURL=fileCache.js.map