unpak.js
Version:
Modern TypeScript library for reading Unreal Engine pak files and assets, inspired by CUE4Parse
332 lines • 11.1 kB
JavaScript
"use strict";
/**
* Phase 10: Advanced File Systems - Virtual File System implementation
*
* Provides a unified interface for accessing files from multiple archives
* with intelligent caching, prioritization, and async loading capabilities.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.VirtualFileSystem = exports.LoadPriority = void 0;
const collection_1 = require("@discordjs/collection");
/**
* Priority levels for file loading
*/
var LoadPriority;
(function (LoadPriority) {
LoadPriority[LoadPriority["LOW"] = 0] = "LOW";
LoadPriority[LoadPriority["NORMAL"] = 1] = "NORMAL";
LoadPriority[LoadPriority["HIGH"] = 2] = "HIGH";
LoadPriority[LoadPriority["CRITICAL"] = 3] = "CRITICAL";
})(LoadPriority || (exports.LoadPriority = LoadPriority = {}));
/**
* Virtual File System for multi-archive support
*
* Features:
* - Mount multiple archives with priority system
* - Intelligent LRU caching with size limits
* - Asynchronous loading with priority queues
* - File override system for modding
* - Performance monitoring and statistics
*/
class VirtualFileSystem {
mounts = new collection_1.Collection();
cache = new collection_1.Collection();
loadQueue = [];
activeLoads = new Set();
config;
// Statistics
stats = {
cacheHits: 0,
cacheMisses: 0,
totalLoads: 0,
totalCacheSize: 0,
lastStatsReport: Date.now()
};
constructor(config = {}) {
this.config = {
maxCacheSize: config.maxCacheSize ?? 256 * 1024 * 1024, // 256MB
maxCacheEntries: config.maxCacheEntries ?? 1000,
enableLRU: config.enableLRU ?? true,
maxConcurrentLoads: config.maxConcurrentLoads ?? 4,
statsInterval: config.statsInterval ?? 30000
};
// Start periodic stats reporting
if (this.config.statsInterval > 0) {
setInterval(() => this.reportStats(), this.config.statsInterval);
}
}
/**
* Mount an archive at the specified path
*/
mount(mountPath, archive, priority = 0, readOnly = true) {
const mount = {
archive,
mountPath: mountPath.replace(/\\/g, '/'),
priority,
readOnly
};
this.mounts.set(mountPath, mount);
// Re-sort mounts by priority (higher priority first)
const sortedMounts = Array.from(this.mounts.entries())
.sort(([, a], [, b]) => b.priority - a.priority);
this.mounts.clear();
for (const [path, mount] of sortedMounts) {
this.mounts.set(path, mount);
}
}
/**
* Unmount an archive
*/
unmount(mountPath) {
// Clear cache entries from this mount
const normalizedPath = mountPath.replace(/\\/g, '/');
for (const [filePath] of this.cache) {
if (filePath.startsWith(normalizedPath)) {
this.removeFromCache(filePath);
}
}
return this.mounts.delete(normalizedPath);
}
/**
* Get file data synchronously (cache only)
*/
getFileSync(filePath) {
const normalizedPath = this.normalizePath(filePath);
// Check cache first
const cached = this.cache.get(normalizedPath);
if (cached) {
this.stats.cacheHits++;
cached.lastAccess = Date.now();
cached.accessCount++;
return cached.data;
}
this.stats.cacheMisses++;
return null;
}
/**
* Get file data asynchronously with priority support
*/
async getFileAsync(filePath, priority = LoadPriority.NORMAL) {
const normalizedPath = this.normalizePath(filePath);
// Check cache first
const cached = this.getFileSync(normalizedPath);
if (cached) {
return cached;
}
// Check if already loading
if (this.activeLoads.has(normalizedPath)) {
// Wait for existing load
return new Promise((resolve, reject) => {
this.loadQueue.push({
filePath: normalizedPath,
priority,
timestamp: Date.now(),
resolve,
reject
});
});
}
// Start new load
return this.startLoad(normalizedPath, priority);
}
/**
* Check if file exists in any mounted archive
*/
fileExists(filePath) {
const normalizedPath = this.normalizePath(filePath);
// Check cache first
if (this.cache.has(normalizedPath)) {
return true;
}
// Check all mounts
for (const mount of this.mounts.values()) {
const relativePath = this.getRelativePath(normalizedPath, mount.mountPath);
if (relativePath && mount.archive.hasFile(relativePath)) {
return true;
}
}
return false;
}
/**
* List all files matching pattern
*/
listFiles(pattern) {
const allFiles = new Set();
// Add cached files
for (const filePath of this.cache.keys()) {
if (!pattern || pattern.test(filePath)) {
allFiles.add(filePath);
}
}
// Add files from archives
for (const mount of this.mounts.values()) {
try {
const archiveFiles = mount.archive.getFileList();
for (const file of archiveFiles) {
const fullPath = mount.mountPath + '/' + file;
if (!pattern || pattern.test(fullPath)) {
allFiles.add(fullPath);
}
}
}
catch (error) {
// Skip archives that don't support file listing
continue;
}
}
return Array.from(allFiles).sort();
}
/**
* Clear cache
*/
clearCache() {
this.cache.clear();
this.stats.totalCacheSize = 0;
}
/**
* Get VFS statistics
*/
getStats() {
return {
...this.stats,
cacheEntries: this.cache.size,
mountedArchives: this.mounts.size
};
}
// Private methods
async startLoad(filePath, priority) {
this.activeLoads.add(filePath);
this.stats.totalLoads++;
try {
// Find the best mount for this file
const mount = this.findBestMount(filePath);
if (!mount) {
return null;
}
const relativePath = this.getRelativePath(filePath, mount.mountPath);
if (!relativePath) {
return null;
}
// Load the file
const data = await this.loadFromArchive(mount.archive, relativePath);
if (data) {
// Add to cache
this.addToCache(filePath, data, priority);
}
// Process any queued requests for this file
this.processQueuedRequests(filePath, data);
return data;
}
catch (error) {
this.processQueuedRequests(filePath, null, error);
throw error;
}
finally {
this.activeLoads.delete(filePath);
}
}
findBestMount(filePath) {
for (const mount of this.mounts.values()) {
const relativePath = this.getRelativePath(filePath, mount.mountPath);
if (relativePath && mount.archive.hasFile(relativePath)) {
return mount;
}
}
return null;
}
async loadFromArchive(archive, filePath) {
try {
// Use getFile directly as it already returns Promise<Buffer | null>
const data = await archive.getFile(filePath);
return data;
}
catch (error) {
return null;
}
}
addToCache(filePath, data, priority) {
// Check if we need to evict entries
if (this.config.enableLRU) {
this.evictIfNeeded(data.length);
}
const entry = {
data,
lastAccess: Date.now(),
size: data.length,
accessCount: 1,
priority
};
this.cache.set(filePath, entry);
this.stats.totalCacheSize += data.length;
}
evictIfNeeded(newDataSize) {
// Check size limit
while (this.stats.totalCacheSize + newDataSize > this.config.maxCacheSize && this.cache.size > 0) {
this.evictLRUEntry();
}
// Check entry count limit
while (this.cache.size >= this.config.maxCacheEntries) {
this.evictLRUEntry();
}
}
evictLRUEntry() {
let oldestEntry = null;
let oldestTime = Date.now();
for (const [filePath, entry] of this.cache) {
// Prefer lower priority and older access time
const score = entry.lastAccess - (entry.priority * 3600000); // Priority worth 1 hour
if (score < oldestTime) {
oldestTime = score;
oldestEntry = filePath;
}
}
if (oldestEntry) {
this.removeFromCache(oldestEntry);
}
}
removeFromCache(filePath) {
const entry = this.cache.get(filePath);
if (entry) {
this.stats.totalCacheSize -= entry.size;
return this.cache.delete(filePath);
}
return false;
}
processQueuedRequests(filePath, data, error) {
const matchingRequests = this.loadQueue.filter(req => req.filePath === filePath);
this.loadQueue = this.loadQueue.filter(req => req.filePath !== filePath);
for (const request of matchingRequests) {
if (error) {
request.reject(error);
}
else {
request.resolve(data);
}
}
}
normalizePath(filePath) {
return filePath.replace(/\\/g, '/').toLowerCase();
}
getRelativePath(fullPath, mountPath) {
const normalizedFull = this.normalizePath(fullPath);
const normalizedMount = this.normalizePath(mountPath);
if (normalizedFull.startsWith(normalizedMount)) {
return normalizedFull.substring(normalizedMount.length).replace(/^\/+/, '');
}
return null;
}
reportStats() {
const now = Date.now();
const elapsed = now - this.stats.lastStatsReport;
if (elapsed > 0) {
const hitRate = this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses) * 100;
console.log(`[VFS Stats] Cache Hit Rate: ${hitRate.toFixed(1)}%, ` +
`Size: ${(this.stats.totalCacheSize / 1024 / 1024).toFixed(1)}MB, ` +
`Entries: ${this.cache.size}, ` +
`Active Loads: ${this.activeLoads.size}`);
}
this.stats.lastStatsReport = now;
}
}
exports.VirtualFileSystem = VirtualFileSystem;
//# sourceMappingURL=VirtualFileSystem.js.map