datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
297 lines • 11.2 kB
JavaScript
"use strict";
/**
* Performance monitoring for DataPilot CLI
* Tracks memory usage, CPU, throughput, and other metrics
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.PerformanceMonitor = void 0;
const perf_hooks_1 = require("perf_hooks");
const process_1 = require("process");
const os = __importStar(require("os"));
class PerformanceMonitor {
startTime = 0;
metrics = [];
phaseMarkers = new Map();
monitoringInterval;
lastCpuUsage;
totalRowsProcessed = 0;
enabled;
sampleInterval;
memoryThreshold;
constructor(options = {}) {
this.enabled = options.enabled ?? true;
this.sampleInterval = options.sampleInterval ?? 100; // ms
this.memoryThreshold = options.memoryThreshold ?? 0.8; // 80% of available memory
}
/**
* Start performance monitoring
*/
start() {
if (!this.enabled)
return;
this.startTime = perf_hooks_1.performance.now();
this.lastCpuUsage = (0, process_1.cpuUsage)();
this.metrics = [];
// Take initial reading
this.collectMetrics();
// Start periodic monitoring
this.monitoringInterval = setInterval(() => {
this.collectMetrics();
this.checkThresholds();
}, this.sampleInterval);
}
/**
* Stop performance monitoring
*/
stop() {
if (this.monitoringInterval) {
clearInterval(this.monitoringInterval);
this.monitoringInterval = undefined;
}
// Take final reading
this.collectMetrics();
return this.generateReport();
}
/**
* Mark the start of a phase
*/
startPhase(phaseName) {
if (!this.enabled)
return;
const currentMetrics = this.getCurrentMetrics();
this.phaseMarkers.set(phaseName, {
start: perf_hooks_1.performance.now(),
startMetrics: currentMetrics,
});
}
/**
* Mark the end of a phase
*/
endPhase(phaseName, rowsProcessed) {
if (!this.enabled)
return;
const phaseData = this.phaseMarkers.get(phaseName);
if (!phaseData)
return;
const endMetrics = this.getCurrentMetrics();
const duration = perf_hooks_1.performance.now() - phaseData.start;
if (rowsProcessed) {
this.totalRowsProcessed += rowsProcessed;
}
// Store phase metrics for report
const phaseMetrics = {
duration,
memoryDelta: endMetrics.memory.heapUsed - phaseData.startMetrics.memory.heapUsed,
cpuTime: endMetrics.cpu.user +
endMetrics.cpu.system -
(phaseData.startMetrics.cpu.user + phaseData.startMetrics.cpu.system),
rowsProcessed,
};
// We'll store these in the metrics array with phase markers
this.metrics.push({
...endMetrics,
phase: { name: phaseName, metrics: phaseMetrics },
});
}
/**
* Update row processing count
*/
updateRowCount(rows) {
this.totalRowsProcessed += rows;
}
/**
* Collect current performance metrics
*/
collectMetrics() {
const currentMetrics = this.getCurrentMetrics();
this.metrics.push(currentMetrics);
}
/**
* Get current performance metrics
*/
getCurrentMetrics() {
const mem = (0, process_1.memoryUsage)();
const cpu = this.getCpuUsage();
return {
timestamp: perf_hooks_1.performance.now() - this.startTime,
memory: {
heapUsed: mem.heapUsed,
heapTotal: mem.heapTotal,
external: mem.external,
rss: mem.rss,
},
cpu,
throughput: this.calculateThroughput(),
};
}
/**
* Calculate CPU usage percentage
*/
getCpuUsage() {
const usage = (0, process_1.cpuUsage)();
let percent = 0;
if (this.lastCpuUsage && this.metrics.length > 0) {
const userDelta = usage.user - this.lastCpuUsage.user;
const systemDelta = usage.system - this.lastCpuUsage.system;
const timeDelta = (perf_hooks_1.performance.now() - this.startTime) * 1000; // Convert to microseconds
percent = ((userDelta + systemDelta) / timeDelta) * 100;
}
this.lastCpuUsage = usage;
return {
user: usage.user / 1000000, // Convert to seconds
system: usage.system / 1000000,
percent: Math.min(percent, 100 * os.cpus().length), // Cap at number of CPUs * 100
};
}
/**
* Calculate current throughput
*/
calculateThroughput() {
if (this.totalRowsProcessed === 0 || this.metrics.length === 0) {
return undefined;
}
const duration = (perf_hooks_1.performance.now() - this.startTime) / 1000; // seconds
const rowsPerSecond = this.totalRowsProcessed / duration;
// Estimate bytes per second (rough approximation)
const avgRowSize = 100; // bytes (estimate)
const bytesPerSecond = rowsPerSecond * avgRowSize;
return {
rowsPerSecond: Math.round(rowsPerSecond),
bytesPerSecond: Math.round(bytesPerSecond),
};
}
/**
* Check performance thresholds and emit warnings
*/
checkThresholds() {
const currentMetrics = this.metrics[this.metrics.length - 1];
if (!currentMetrics)
return;
const totalMemory = os.totalmem();
const memoryUsagePercent = currentMetrics.memory.rss / totalMemory;
if (memoryUsagePercent > this.memoryThreshold) {
console.warn(`⚠️ High memory usage: ${(memoryUsagePercent * 100).toFixed(1)}% of system memory`);
}
// Check if memory is growing rapidly
if (this.metrics.length > 10) {
const recentMetrics = this.metrics.slice(-10);
const memoryGrowth = recentMetrics[9].memory.heapUsed - recentMetrics[0].memory.heapUsed;
const timeSpan = recentMetrics[9].timestamp - recentMetrics[0].timestamp;
const growthRate = memoryGrowth / timeSpan; // bytes per ms
if (growthRate > 1000000) {
// 1MB per ms
console.warn(`⚠️ Rapid memory growth detected: ${(growthRate / 1000000).toFixed(1)} MB/s`);
}
}
}
/**
* Generate performance report
*/
generateReport() {
const duration = perf_hooks_1.performance.now() - this.startTime;
const warnings = [];
// Calculate summary metrics
const memoryMetrics = this.metrics.map((m) => m.memory.heapUsed);
const cpuMetrics = this.metrics.map((m) => m.cpu.percent);
const peakMemoryMB = Math.max(...memoryMetrics) / 1024 / 1024;
const avgMemoryMB = memoryMetrics.reduce((a, b) => a + b, 0) / memoryMetrics.length / 1024 / 1024;
const avgCpuPercent = cpuMetrics.reduce((a, b) => a + b, 0) / cpuMetrics.length;
// Check for performance issues
if (peakMemoryMB > 1024) {
warnings.push(`Peak memory usage exceeded 1GB: ${peakMemoryMB.toFixed(0)}MB`);
}
if (avgCpuPercent > 80) {
warnings.push(`High average CPU usage: ${avgCpuPercent.toFixed(1)}%`);
}
// Extract phase information
const phases = {};
this.metrics.forEach((metric) => {
if (metric.phase) {
phases[metric.phase.name] = metric.phase.metrics;
}
});
return {
summary: {
totalDuration: duration,
peakMemoryMB,
avgMemoryMB,
avgCpuPercent,
totalRowsProcessed: this.totalRowsProcessed,
avgThroughput: this.totalRowsProcessed / (duration / 1000),
},
phases,
warnings,
timeline: this.metrics,
};
}
/**
* Format performance report for display
*/
static formatReport(report) {
const lines = [];
lines.push('📊 Performance Report');
lines.push('='.repeat(50));
lines.push('\n📈 Summary:');
lines.push(` Total Duration: ${(report.summary.totalDuration / 1000).toFixed(2)}s`);
lines.push(` Peak Memory: ${report.summary.peakMemoryMB.toFixed(1)} MB`);
lines.push(` Avg Memory: ${report.summary.avgMemoryMB.toFixed(1)} MB`);
lines.push(` Avg CPU: ${report.summary.avgCpuPercent.toFixed(1)}%`);
lines.push(` Rows Processed: ${report.summary.totalRowsProcessed.toLocaleString()}`);
lines.push(` Throughput: ${report.summary.avgThroughput.toFixed(0)} rows/s`);
if (Object.keys(report.phases).length > 0) {
lines.push('\n⏱️ Phase Breakdown:');
for (const [phase, metrics] of Object.entries(report.phases)) {
lines.push(` ${phase}:`);
lines.push(` Duration: ${(metrics.duration / 1000).toFixed(2)}s`);
lines.push(` Memory Δ: ${(metrics.memoryDelta / 1024 / 1024).toFixed(1)} MB`);
if (metrics.rowsProcessed) {
lines.push(` Rows: ${metrics.rowsProcessed.toLocaleString()}`);
}
}
}
if (report.warnings.length > 0) {
lines.push('\n⚠️ Warnings:');
report.warnings.forEach((warning) => {
lines.push(` • ${warning}`);
});
}
return lines.join('\n');
}
}
exports.PerformanceMonitor = PerformanceMonitor;
// Export types
// Types are already exported above
//# sourceMappingURL=performance-monitor.js.map