datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
312 lines ā¢ 12.5 kB
JavaScript
;
/**
* Performance Manager for DataPilot CLI
* Integrates SmartResourceManager and SectionCacheManager for optimal performance
* Addresses GitHub issue #23: Smart performance defaults and auto-configuration
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.CLIPerformanceManager = void 0;
const smart_resource_manager_1 = require("../performance/smart-resource-manager");
const section_cache_manager_1 = require("../performance/section-cache-manager");
const performance_monitor_1 = require("./performance-monitor");
const logger_1 = require("../utils/logger");
const fs = __importStar(require("fs"));
class CLIPerformanceManager {
resourceManager;
cacheManager;
performanceMonitor;
currentConfig;
constructor() {
this.resourceManager = new smart_resource_manager_1.SmartResourceManager();
}
/**
* Initialize performance systems based on CLI options and file characteristics
*/
async initialize(filePath, options) {
logger_1.logger.info('Initializing smart performance management...');
// Get file size for optimization
const fileSizeMB = await this.getFileSizeMB(filePath);
// Auto-configure if requested or no specific performance options provided
const shouldAutoConfig = options.autoConfig || this.shouldUseAutoConfig(options);
let autoConfig;
if (shouldAutoConfig) {
autoConfig = await this.resourceManager.autoConfigurePerformance(fileSizeMB);
this.logAutoConfigResults(autoConfig);
}
else {
// Use manual configuration
autoConfig = await this.createManualConfig(options, fileSizeMB);
}
// Apply preset if specified
if (options.preset) {
autoConfig = await this.applyPreset(options.preset, fileSizeMB);
}
// Initialize caching system
const cacheConfig = this.createCacheConfig(options, autoConfig);
this.cacheManager = new section_cache_manager_1.SectionCacheManager(cacheConfig);
// Initialize performance monitoring
this.performanceMonitor = new performance_monitor_1.PerformanceMonitor({
enabled: !options.quiet,
sampleInterval: 100,
memoryThreshold: 0.8,
});
// Create optimized CLI options
const optimizedOptions = this.createOptimizedOptions(options, autoConfig);
this.currentConfig = {
autoConfiguration: autoConfig,
cacheManager: this.cacheManager,
performanceMonitor: this.performanceMonitor,
optimizedOptions,
};
return this.currentConfig;
}
/**
* Start performance monitoring
*/
startMonitoring() {
if (this.performanceMonitor) {
this.performanceMonitor.start();
}
}
/**
* Stop performance monitoring and get report
*/
stopMonitoring() {
if (this.performanceMonitor) {
return this.performanceMonitor.stop();
}
return undefined;
}
/**
* Get section result from cache if available
*/
async getCachedResult(filePath, section) {
if (!this.cacheManager)
return null;
return await this.cacheManager.get(filePath, section);
}
/**
* Cache section result
*/
async setCachedResult(filePath, section, data) {
if (this.cacheManager) {
await this.cacheManager.set(filePath, section, data);
}
}
/**
* Check if caching is enabled
*/
isCachingEnabled() {
return Boolean(this.cacheManager);
}
/**
* Clear cache for a specific file
*/
async clearFileCache(filePath) {
if (this.cacheManager) {
await this.cacheManager.clearFile(filePath);
}
}
/**
* Get cache statistics
*/
async getCacheStats() {
if (this.cacheManager) {
return await this.cacheManager.getStats();
}
return null;
}
/**
* Get current performance configuration
*/
getCurrentConfig() {
return this.currentConfig;
}
/**
* Get recommended CLI options for a file
*/
async getRecommendedOptions(filePath) {
const fileSizeMB = await this.getFileSizeMB(filePath);
return await this.resourceManager.getRecommendedCLIOptions(fileSizeMB);
}
/**
* Check if auto-configuration should be used
*/
shouldUseAutoConfig(options) {
// Use auto-config if no specific performance options are set
return !options.maxMemory &&
!options.threads &&
!options.preset &&
options.enableCaching === undefined;
}
/**
* Get file size in MB
*/
async getFileSizeMB(filePath) {
try {
const stats = await fs.promises.stat(filePath);
return stats.size / (1024 * 1024);
}
catch (error) {
logger_1.logger.warn(`Could not determine file size for ${filePath}, using default optimization`);
return 100; // Default assumption
}
}
/**
* Create manual configuration from CLI options
*/
async createManualConfig(options, fileSizeMB) {
const resources = await this.resourceManager.detectSystemResources();
return {
detectedResources: resources,
selectedProfile: {
name: 'manual',
description: 'Manually configured via CLI options',
maxMemoryMB: options.maxMemory || 2048,
chunkSize: options.chunkSize || 10000,
enableParallel: options.threads ? options.threads > 1 : true,
enableCaching: options.enableCaching ?? true,
maxWorkers: options.threads || 4,
aggressiveCleanup: false,
streamingOptimizations: options.streamingOptimizations || false,
},
customizations: {
maxMemoryMB: options.maxMemory || 2048,
chunkSize: options.chunkSize || 10000,
maxWorkers: options.threads || 4,
enableCaching: options.enableCaching ?? true,
streamingOptimizations: options.streamingOptimizations || false,
progressiveReporting: options.progressiveReporting || false,
},
recommendations: [`Using manual configuration from CLI options`],
warnings: [],
};
}
/**
* Apply performance preset
*/
async applyPreset(preset, fileSizeMB) {
logger_1.logger.info(`Applying performance preset: ${preset}`);
return await this.resourceManager.autoConfigurePerformance(fileSizeMB);
}
/**
* Create cache configuration
*/
createCacheConfig(options, autoConfig) {
const enabled = options.enableCaching ?? autoConfig.customizations.enableCaching;
return {
enabled,
maxSizeBytes: (options.cacheSize || autoConfig.customizations.cacheSize || 500) * 1024 * 1024,
maxEntries: 1000,
ttlMs: 24 * 60 * 60 * 1000, // 24 hours
cacheDirectory: require('path').join(require('os').tmpdir(), 'datapilot-cache'),
enableDiskCache: enabled,
enableMemoryCache: enabled,
compressionLevel: 6,
};
}
/**
* Create optimized CLI options
*/
createOptimizedOptions(options, autoConfig) {
return {
...options,
maxMemory: options.maxMemory || autoConfig.customizations.maxMemoryMB,
chunkSize: options.chunkSize || autoConfig.customizations.chunkSize,
threads: options.threads || autoConfig.customizations.maxWorkers,
enableCaching: options.enableCaching ?? autoConfig.customizations.enableCaching,
streamingOptimizations: options.streamingOptimizations || autoConfig.customizations.streamingOptimizations,
progressiveReporting: options.progressiveReporting || autoConfig.customizations.progressiveReporting || false,
};
}
/**
* Log auto-configuration results
*/
logAutoConfigResults(autoConfig) {
if (!autoConfig.recommendations.length && !autoConfig.warnings.length) {
return;
}
logger_1.logger.info('š Smart Performance Auto-Configuration Applied');
if (autoConfig.recommendations.length > 0) {
logger_1.logger.info('š” Performance Recommendations:');
autoConfig.recommendations.forEach(rec => logger_1.logger.info(` ā¢ ${rec}`));
}
if (autoConfig.warnings.length > 0) {
logger_1.logger.warn('ā ļø Performance Warnings:');
autoConfig.warnings.forEach(warn => logger_1.logger.warn(` ā¢ ${warn}`));
}
}
/**
* Show performance dashboard
*/
showDashboard() {
if (!this.currentConfig) {
console.log('ā Performance manager not initialized');
return;
}
const { autoConfiguration: config } = this.currentConfig;
const resources = config.detectedResources;
console.log('\nš DataPilot Performance Dashboard');
console.log('='.repeat(50));
console.log('\nš„ļø System Resources:');
console.log(` Platform: ${resources.platform}/${resources.arch}`);
console.log(` CPU Cores: ${resources.cpuCount}`);
console.log(` Total Memory: ${resources.totalMemoryGB.toFixed(1)} GB`);
console.log(` Available Memory: ${resources.freeMemoryGB.toFixed(1)} GB`);
console.log(` Load Average: ${resources.cpuLoadAverage.toFixed(2)}`);
console.log('\nā” Active Configuration:');
console.log(` Profile: ${config.selectedProfile.name}`);
console.log(` Memory Limit: ${config.customizations.maxMemoryMB} MB`);
console.log(` Chunk Size: ${config.customizations.chunkSize} rows`);
console.log(` Workers: ${config.customizations.maxWorkers}`);
console.log(` Caching: ${config.customizations.enableCaching ? 'Enabled' : 'Disabled'}`);
console.log(` Streaming: ${config.customizations.streamingOptimizations ? 'Enabled' : 'Disabled'}`);
if (this.cacheManager) {
this.cacheManager.getStats().then(stats => {
console.log('\nš¾ Cache Statistics:');
console.log(` Total Entries: ${stats.totalEntries}`);
console.log(` Cache Size: ${(stats.totalSizeBytes / 1024 / 1024).toFixed(1)} MB`);
console.log(` Hit Rate: ${(stats.hitRate * 100).toFixed(1)}%`);
console.log(` Total Hits: ${stats.totalHits}`);
console.log(` Total Misses: ${stats.totalMisses}`);
}).catch(() => {
console.log('\nš¾ Cache Statistics: Not available');
});
}
}
}
exports.CLIPerformanceManager = CLIPerformanceManager;
//# sourceMappingURL=performance-manager.js.map