packfs-core
Version:
Semantic filesystem operations for LLM agent frameworks with natural language understanding. See LLM_AGENT_GUIDE.md for copy-paste examples.
293 lines • 10.7 kB
JavaScript
"use strict";
/**
* Compression Engine for PackFS - Orchestrates multiple compression strategies
*
* This engine manages the selection and application of appropriate compression
* algorithms based on file type, access patterns, and system resources.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.CompressionProfiles = exports.CompressionEngine = void 0;
const CompressionStrategy_1 = require("./CompressionStrategy");
const BrotliStrategy_1 = require("./BrotliStrategy");
const LZ4Strategy_1 = require("./LZ4Strategy");
const ZstdStrategy_1 = require("./ZstdStrategy");
/**
* Main compression engine that orchestrates multiple strategies
*/
class CompressionEngine {
constructor(profile) {
this.compressionStats = new Map();
this.profile = profile;
this.registry = new CompressionStrategy_1.StrategyRegistry();
// Initialize default strategies
this.initializeStrategies();
}
/**
* Compress data using the optimal strategy
*/
async compress(data, mimeType, metadata) {
const startTime = performance.now();
try {
// Build compression hints
const hints = {
mimeType,
accessFrequency: metadata?.accessFrequency || 0.5,
fileSize: data.length,
isHot: this.isHotFile(metadata?.path, metadata?.accessFrequency),
ecosystem: this.detectEcosystem(mimeType, metadata?.path)
};
// Select optimal strategy
const strategy = this.registry.getOptimal(data, hints);
// Apply compression
const chunk = await strategy.compress(data, hints);
const compressionTime = performance.now() - startTime;
const compressionRatio = chunk.compressedSize / chunk.originalSize;
// Update statistics
this.updateStats(strategy.name, {
compressionRatio,
compressionTime,
originalSize: chunk.originalSize,
compressedSize: chunk.compressedSize
});
return {
success: true,
algorithm: strategy.name,
originalSize: chunk.originalSize,
compressedSize: chunk.compressedSize,
compressionRatio,
compressionTime,
chunk
};
}
catch (error) {
return {
success: false,
algorithm: 'none',
originalSize: data.length,
compressedSize: data.length,
compressionRatio: 1,
compressionTime: performance.now() - startTime,
error: error instanceof Error ? error.message : 'Unknown compression error'
};
}
}
/**
* Decompress a compressed chunk
*/
async decompress(chunk) {
const strategy = this.registry.get(chunk.algorithm);
if (!strategy) {
throw new Error(`Unknown compression algorithm: ${chunk.algorithm}`);
}
return strategy.decompress(chunk);
}
/**
* Create a streaming decompressor
*/
createDecompressor(chunk) {
const strategy = this.registry.get(chunk.algorithm);
if (!strategy || !strategy.supportsStreaming) {
return null;
}
return strategy.createDecompressor(chunk);
}
/**
* Analyze which compression strategy would be best for given data
*/
analyzeOptimalStrategy(data, mimeType, metadata) {
const hints = {
mimeType,
accessFrequency: metadata?.accessFrequency || 0.5,
fileSize: data.length,
isHot: this.isHotFile(metadata?.path, metadata?.accessFrequency),
ecosystem: this.detectEcosystem(mimeType, metadata?.path)
};
const strategies = Array.from(this.registry['strategies'].values());
const analysis = {
recommendedStrategy: '',
estimations: []
};
for (const strategy of strategies) {
if (strategy.shouldUse(data, hints)) {
const estimation = {
algorithm: strategy.name,
estimatedRatio: strategy.estimateRatio(data, hints),
priority: strategy.priority,
suitable: true
};
analysis.estimations.push(estimation);
}
}
// Sort by priority for hot files, otherwise by compression ratio
if (hints.isHot) {
// For hot files, prioritize speed
analysis.estimations.sort((a, b) => {
const priorityOrder = { speed: 0, balanced: 1, ratio: 2 };
return (priorityOrder[a.priority] || 999) - (priorityOrder[b.priority] || 999);
});
}
else {
// For cold files, prioritize compression ratio
analysis.estimations.sort((a, b) => a.estimatedRatio - b.estimatedRatio);
}
analysis.recommendedStrategy = analysis.estimations[0]?.algorithm || 'none';
return analysis;
}
/**
* Get compression statistics
*/
getStatistics() {
const stats = {
totalCompressions: 0,
totalDecompressions: 0,
totalBytesProcessed: 0,
totalBytesSaved: 0,
averageCompressionRatio: 0,
averageCompressionTime: 0,
strategyUsage: {}
};
for (const [algorithm, algorithmStats] of Array.from(this.compressionStats)) {
stats.totalCompressions += algorithmStats.compressionCount;
stats.totalBytesProcessed += algorithmStats.totalOriginalSize;
stats.totalBytesSaved += algorithmStats.totalOriginalSize - algorithmStats.totalCompressedSize;
stats.strategyUsage[algorithm] = algorithmStats.compressionCount;
}
if (stats.totalBytesProcessed > 0) {
stats.averageCompressionRatio = 1 - (stats.totalBytesSaved / stats.totalBytesProcessed);
}
return stats;
}
/**
* Initialize compression strategies based on profile
*/
initializeStrategies() {
// Always register LZ4 for speed
this.registry.register(new LZ4Strategy_1.LZ4Strategy());
// Register Brotli for text compression
if (!this.profile.prioritizeSpeed || this.profile.maxMemoryUsage > 256 * 1024 * 1024) {
const brotliDictionary = this.profile.enableDictionary ? this.loadDictionary('javascript') : undefined;
this.registry.register(new BrotliStrategy_1.BrotliStrategy(brotliDictionary));
}
// Register Zstd for balanced compression
if (!this.profile.development) {
this.registry.register(new ZstdStrategy_1.ZstdStrategy(this.profile.prioritizeSpeed ? 1 : 3));
}
// Add custom strategies from profile
for (const [, strategy] of Object.entries(this.profile.strategies)) {
this.registry.register(strategy);
}
}
/**
* Detect if a file is "hot" (frequently accessed)
*/
isHotFile(path, accessFrequency) {
if (accessFrequency !== undefined && accessFrequency > 0.8) {
return true;
}
// Common hot file patterns
if (path) {
const hotPatterns = [
/node_modules\/.*\/(index|main)\.(js|ts)$/,
/package\.json$/,
/tsconfig\.json$/,
/\.env$/,
/src\/.*\.(js|ts|jsx|tsx)$/
];
return hotPatterns.some(pattern => pattern.test(path));
}
return false;
}
/**
* Detect the ecosystem based on file type and path
*/
detectEcosystem(_mimeType, path) {
if (path) {
if (path.includes('react') || path.endsWith('.jsx') || path.endsWith('.tsx')) {
return 'react';
}
if (path.includes('vue') || path.endsWith('.vue')) {
return 'vue';
}
if (path.includes('angular') || path.includes('.component.')) {
return 'angular';
}
if (path.includes('node_modules') || path.includes('server')) {
return 'node';
}
}
return 'unknown';
}
/**
* Load compression dictionary for specific ecosystem
*/
loadDictionary(ecosystem) {
// In production, this would load actual dictionary files
// For now, return a mock dictionary
const dictionaries = {
javascript: 'const function export import async await class extends',
react: 'useState useEffect useContext props children component render',
vue: 'template script style computed methods mounted created watch',
angular: 'component service module injectable pipe directive template'
};
const dict = dictionaries[ecosystem];
return dict ? Buffer.from(dict) : undefined;
}
/**
* Update compression statistics
*/
updateStats(algorithm, result) {
const stats = this.compressionStats.get(algorithm) || {
compressionCount: 0,
decompressionCount: 0,
totalOriginalSize: 0,
totalCompressedSize: 0,
totalCompressionTime: 0,
totalDecompressionTime: 0
};
stats.compressionCount++;
stats.totalOriginalSize += result.originalSize;
stats.totalCompressedSize += result.compressedSize;
stats.totalCompressionTime += result.compressionTime;
this.compressionStats.set(algorithm, stats);
}
}
exports.CompressionEngine = CompressionEngine;
/**
* Pre-configured compression profiles
*/
exports.CompressionProfiles = {
/**
* Development profile - prioritize speed
*/
development: {
name: 'development',
development: true,
maxMemoryUsage: 256 * 1024 * 1024,
prioritizeSpeed: true,
enableDictionary: false,
strategies: {}
},
/**
* Production profile - balanced performance
*/
production: {
name: 'production',
development: false,
maxMemoryUsage: 512 * 1024 * 1024,
prioritizeSpeed: false,
enableDictionary: true,
strategies: {}
},
/**
* CI profile - minimize resource usage
*/
ci: {
name: 'ci',
development: false,
maxMemoryUsage: 128 * 1024 * 1024,
prioritizeSpeed: false,
enableDictionary: false,
strategies: {}
}
};
//# sourceMappingURL=CompressionEngine.js.map