unpak.js
Version:
Modern TypeScript library for reading Unreal Engine pak files and assets, inspired by CUE4Parse
364 lines • 12.5 kB
JavaScript
"use strict";
/**
* Phase 10: Advanced File Systems - Async Loading Manager
*
* Manages prioritized async loading of assets with worker thread support
* and intelligent preloading based on usage patterns.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.AsyncLoadingManager = void 0;
const VirtualFileSystem_1 = require("./VirtualFileSystem");
const events_1 = require("events");
/**
* Async Loading Manager with intelligent preloading
*
* Features:
* - Priority-based job queue
* - Batch loading with progress tracking
* - Intelligent preloading based on patterns
* - Load balancing across multiple worker threads
* - Performance metrics and optimization
*/
class AsyncLoadingManager extends events_1.EventEmitter {
vfs;
jobQueue = [];
activeJobs = new Map();
completedJobs = new Map();
preloadPatterns = [];
maxConcurrentJobs;
isProcessing = false;
// Usage tracking for intelligent preloading
accessPatterns = new Map();
lastAccessTime = new Map();
// Performance metrics
metrics = {
totalJobs: 0,
completedJobs: 0,
failedJobs: 0,
totalLoadTime: 0,
averageLoadTime: 0,
preloadHits: 0,
preloadMisses: 0
};
constructor(vfs, maxConcurrentJobs = 4) {
super();
this.vfs = vfs;
this.maxConcurrentJobs = maxConcurrentJobs;
// Setup default preload patterns
this.setupDefaultPreloadPatterns();
// Start processing jobs
this.processJobs();
}
/**
* Queue a loading job
*/
queueJob(filePaths, priority = VirtualFileSystem_1.LoadPriority.NORMAL, metadata) {
const jobId = this.generateJobId();
const job = {
id: jobId,
filePaths: [...filePaths],
priority,
createdAt: Date.now(),
metadata
};
this.jobQueue.push(job);
this.sortJobQueue();
this.metrics.totalJobs++;
this.emit('jobQueued', job);
return jobId;
}
/**
* Load files synchronously (returns immediately available data)
*/
loadSync(filePaths) {
const result = new Map();
for (const filePath of filePaths) {
const data = this.vfs.getFileSync(filePath);
if (data) {
result.set(filePath, data);
this.trackAccess(filePath);
}
}
return result;
}
/**
* Load files asynchronously with priority
*/
async loadAsync(filePaths, priority = VirtualFileSystem_1.LoadPriority.NORMAL) {
const jobId = this.queueJob(filePaths, priority);
return new Promise((resolve, reject) => {
const onComplete = (result) => {
if (result.jobId === jobId) {
this.removeListener('jobCompleted', onComplete);
this.removeListener('jobFailed', onFailed);
if (result.success) {
resolve(result.data);
}
else {
reject(result.error || new Error('Load job failed'));
}
}
};
const onFailed = (result) => {
if (result.jobId === jobId) {
this.removeListener('jobCompleted', onComplete);
this.removeListener('jobFailed', onFailed);
reject(result.error || new Error('Load job failed'));
}
};
this.on('jobCompleted', onComplete);
this.on('jobFailed', onFailed);
});
}
/**
* Add a preload pattern
*/
addPreloadPattern(pattern) {
this.preloadPatterns.push(pattern);
}
/**
* Remove a preload pattern
*/
removePreloadPattern(name) {
const index = this.preloadPatterns.findIndex(p => p.name === name);
if (index >= 0) {
this.preloadPatterns.splice(index, 1);
return true;
}
return false;
}
/**
* Trigger preloading based on file access
*/
triggerPreload(triggerFile) {
for (const pattern of this.preloadPatterns) {
if (pattern.triggers.some(trigger => triggerFile.includes(trigger))) {
this.executePreload(pattern);
}
}
}
/**
* Get job status
*/
getJobStatus(jobId) {
if (this.completedJobs.has(jobId)) {
const result = this.completedJobs.get(jobId);
return result.success ? 'completed' : 'failed';
}
if (this.activeJobs.has(jobId)) {
return 'active';
}
if (this.jobQueue.some(job => job.id === jobId)) {
return 'queued';
}
return 'not_found';
}
/**
* Cancel a queued job
*/
cancelJob(jobId) {
const queueIndex = this.jobQueue.findIndex(job => job.id === jobId);
if (queueIndex >= 0) {
this.jobQueue.splice(queueIndex, 1);
this.emit('jobCancelled', jobId);
return true;
}
return false;
}
/**
* Get performance metrics
*/
getMetrics() {
return {
...this.metrics,
queueLength: this.jobQueue.length,
activeJobs: this.activeJobs.size
};
}
/**
* Clear completed job history
*/
clearHistory() {
this.completedJobs.clear();
}
// Private methods
async processJobs() {
if (this.isProcessing) {
return;
}
this.isProcessing = true;
while (true) {
// Check if we can start more jobs
if (this.activeJobs.size >= this.maxConcurrentJobs || this.jobQueue.length === 0) {
await this.sleep(10); // Wait 10ms before checking again
continue;
}
// Get highest priority job
const job = this.jobQueue.shift();
if (!job) {
continue;
}
// Start the job
this.activeJobs.set(job.id, job);
this.emit('jobStarted', job);
// Process job asynchronously
this.processJob(job).catch(error => {
console.error(`Error processing job ${job.id}:`, error);
this.completeJob(job, false, new Map(), job.filePaths, error);
});
}
}
async processJob(job) {
const startTime = Date.now();
const data = new Map();
const failed = [];
try {
// Load files in parallel batches
const batchSize = 10; // Process 10 files at a time
for (let i = 0; i < job.filePaths.length; i += batchSize) {
const batch = job.filePaths.slice(i, i + batchSize);
const batchPromises = batch.map(async (filePath) => {
try {
const fileData = await this.vfs.getFileAsync(filePath, job.priority);
if (fileData) {
data.set(filePath, fileData);
this.trackAccess(filePath);
return { filePath, success: true };
}
else {
failed.push(filePath);
return { filePath, success: false };
}
}
catch (error) {
failed.push(filePath);
return { filePath, success: false, error };
}
});
const batchResults = await Promise.all(batchPromises);
// Emit progress
const progress = (i + batch.length) / job.filePaths.length;
this.emit('jobProgress', {
jobId: job.id,
progress,
loaded: data.size,
failed: failed.length,
total: job.filePaths.length
});
}
const duration = Date.now() - startTime;
const success = failed.length === 0;
this.completeJob(job, success, data, failed, undefined, duration);
}
catch (error) {
const duration = Date.now() - startTime;
this.completeJob(job, false, data, job.filePaths, error, duration);
}
}
completeJob(job, success, data, failed, error, duration) {
const finalDuration = duration || (Date.now() - job.createdAt);
const result = {
jobId: job.id,
success,
data,
failed,
duration: finalDuration,
error
};
this.activeJobs.delete(job.id);
this.completedJobs.set(job.id, result);
// Update metrics
if (success) {
this.metrics.completedJobs++;
}
else {
this.metrics.failedJobs++;
}
this.metrics.totalLoadTime += finalDuration;
this.metrics.averageLoadTime = this.metrics.totalLoadTime / (this.metrics.completedJobs + this.metrics.failedJobs);
// Emit completion event
if (success) {
this.emit('jobCompleted', result);
}
else {
this.emit('jobFailed', result);
}
// Trigger preloading if this was a successful load
if (success && data.size > 0) {
for (const filePath of data.keys()) {
this.triggerPreload(filePath);
}
}
}
sortJobQueue() {
this.jobQueue.sort((a, b) => {
// Higher priority first
if (a.priority !== b.priority) {
return b.priority - a.priority;
}
// Earlier created first for same priority
return a.createdAt - b.createdAt;
});
}
executePreload(pattern) {
try {
const candidateFiles = this.vfs.listFiles();
const matchingFiles = [];
for (const file of candidateFiles) {
if (pattern.patterns.some(regex => regex.test(file))) {
matchingFiles.push(file);
if (pattern.maxFiles && matchingFiles.length >= pattern.maxFiles) {
break;
}
}
}
if (matchingFiles.length > 0) {
// Queue preload job with lower priority
const preloadPriority = Math.max(VirtualFileSystem_1.LoadPriority.LOW, pattern.priority - 1);
this.queueJob(matchingFiles, preloadPriority, { preload: true, pattern: pattern.name });
}
}
catch (error) {
console.warn(`Failed to execute preload pattern ${pattern.name}:`, error);
}
}
trackAccess(filePath) {
const now = Date.now();
this.accessPatterns.set(filePath, (this.accessPatterns.get(filePath) || 0) + 1);
this.lastAccessTime.set(filePath, now);
}
generateJobId() {
return `job_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
setupDefaultPreloadPatterns() {
// Preload related texture files when loading a material
this.addPreloadPattern({
name: 'material_textures',
patterns: [/\.uasset$/i, /\.utexture$/i],
triggers: ['.umaterial', '.umat'],
priority: VirtualFileSystem_1.LoadPriority.NORMAL,
maxFiles: 20
});
// Preload related mesh files when loading a skeletal mesh
this.addPreloadPattern({
name: 'skeletal_mesh_parts',
patterns: [/\.uasset$/i, /\.umesh$/i],
triggers: ['.uskeletalmesh', '.uskel'],
priority: VirtualFileSystem_1.LoadPriority.NORMAL,
maxFiles: 10
});
// Preload audio files when loading a sound bank
this.addPreloadPattern({
name: 'audio_bank_sounds',
patterns: [/\.wem$/i, /\.ogg$/i, /\.wav$/i],
triggers: ['.bnk', '.soundbank'],
priority: VirtualFileSystem_1.LoadPriority.LOW,
maxFiles: 50
});
}
}
exports.AsyncLoadingManager = AsyncLoadingManager;
//# sourceMappingURL=AsyncLoadingManager.js.map