datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
712 lines • 27.6 kB
JavaScript
"use strict";
/**
* Advanced Metrics Collection and Aggregation System for DataPilot
* Collects, processes, and aggregates performance metrics for production monitoring
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.MetricsUtils = exports.globalMetricsCollector = exports.MetricsCollector = exports.MetricType = void 0;
const events_1 = require("events");
const perf_hooks_1 = require("perf_hooks");
const logger_1 = require("../utils/logger");
const performance_monitor_1 = require("../core/performance-monitor");
const memory_manager_1 = require("../utils/memory-manager");
const error_handler_1 = require("../utils/error-handler");
var MetricType;
(function (MetricType) {
MetricType["COUNTER"] = "counter";
MetricType["GAUGE"] = "gauge";
MetricType["HISTOGRAM"] = "histogram";
MetricType["SUMMARY"] = "summary";
})(MetricType || (exports.MetricType = MetricType = {}));
/**
* Production metrics collection and aggregation system
*/
class MetricsCollector extends events_1.EventEmitter {
static instance;
metrics = new Map();
configs = new Map();
aggregations = new Map();
isCollecting = false;
collectionInterval;
aggregationInterval;
startTime = Date.now();
sectionMetrics = new Map();
// Request tracking
requestCount = 0;
errorCount = 0;
responseTimes = [];
dataProcessingStats = {
rowsProcessed: 0,
filesProcessed: 0,
bytesProcessed: 0,
};
constructor() {
super();
this.initializeDefaultMetrics();
this.initializeSectionMetrics();
}
static getInstance() {
if (!MetricsCollector.instance) {
MetricsCollector.instance = new MetricsCollector();
}
return MetricsCollector.instance;
}
/**
* Register a new metric
*/
registerMetric(config) {
this.configs.set(config.name, {
retention: 24 * 60 * 60 * 1000, // 24 hours default
aggregationWindow: 5 * 60 * 1000, // 5 minutes default
...config,
});
this.metrics.set(config.name, {
metric: config.name,
values: [],
retention: config.retention || 24 * 60 * 60 * 1000,
});
logger_1.logger.debug(`Registered metric: ${config.name} (${config.type})`, {
operation: 'registerMetric',
});
}
/**
* Record a metric value
*/
recordMetric(name, value, labels, timestamp) {
const timeSeries = this.metrics.get(name);
if (!timeSeries) {
logger_1.logger.warn(`Metric '${name}' not registered`, { operation: 'recordMetric' });
return;
}
const metricValue = {
value,
timestamp: timestamp || Date.now(),
labels,
};
timeSeries.values.push(metricValue);
this.emit('metricRecorded', name, metricValue);
// Clean up old values
this.cleanupTimeSeries(timeSeries);
}
/**
* Increment a counter metric
*/
incrementCounter(name, amount = 1, labels) {
const config = this.configs.get(name);
if (!config || config.type !== MetricType.COUNTER) {
logger_1.logger.warn(`Counter '${name}' not found or wrong type`, { operation: 'incrementCounter' });
return;
}
this.recordMetric(name, amount, labels);
}
/**
* Set a gauge metric
*/
setGauge(name, value, labels) {
const config = this.configs.get(name);
if (!config || config.type !== MetricType.GAUGE) {
logger_1.logger.warn(`Gauge '${name}' not found or wrong type`, { operation: 'setGauge' });
return;
}
this.recordMetric(name, value, labels);
}
/**
* Record a histogram value
*/
recordHistogram(name, value, labels) {
const config = this.configs.get(name);
if (!config || config.type !== MetricType.HISTOGRAM) {
logger_1.logger.warn(`Histogram '${name}' not found or wrong type`, { operation: 'recordHistogram' });
return;
}
this.recordMetric(name, value, labels);
}
/**
* Record timing information
*/
recordTiming(name, startTime, labels) {
const duration = perf_hooks_1.performance.now() - startTime;
this.recordHistogram(name, duration, labels);
}
/**
* Start timing operation
*/
startTimer(name) {
const startTime = perf_hooks_1.performance.now();
return (labels) => {
this.recordTiming(name, startTime, labels);
};
}
/**
* Record section execution metrics
*/
recordSectionExecution(sectionName, duration, success, context) {
const metrics = this.sectionMetrics.get(sectionName) || {
executionCount: 0,
totalDuration: 0,
avgDuration: 0,
maxDuration: 0,
errorCount: 0,
successRate: 100,
lastExecution: 0,
};
metrics.executionCount++;
metrics.totalDuration += duration;
metrics.avgDuration = metrics.totalDuration / metrics.executionCount;
metrics.maxDuration = Math.max(metrics.maxDuration, duration);
metrics.lastExecution = Date.now();
if (!success) {
metrics.errorCount++;
}
metrics.successRate =
((metrics.executionCount - metrics.errorCount) / metrics.executionCount) * 100;
this.sectionMetrics.set(sectionName, metrics);
// Record as standard metrics
this.recordHistogram('section_duration_ms', duration, { section: sectionName });
this.incrementCounter('section_executions_total', 1, {
section: sectionName,
success: success.toString(),
});
logger_1.logger.debug(`Recorded section execution: ${sectionName} (${duration.toFixed(2)}ms, success: ${success})`, {
...context,
operation: 'recordSectionExecution',
section: sectionName,
duration,
});
}
/**
* Record data processing metrics
*/
recordDataProcessing(rowsProcessed, bytesProcessed, fileProcessed = false) {
this.dataProcessingStats.rowsProcessed += rowsProcessed;
this.dataProcessingStats.bytesProcessed += bytesProcessed;
if (fileProcessed) {
this.dataProcessingStats.filesProcessed++;
}
// Record as standard metrics
this.incrementCounter('rows_processed_total', rowsProcessed);
this.incrementCounter('bytes_processed_total', bytesProcessed);
if (fileProcessed) {
this.incrementCounter('files_processed_total', 1);
}
// Calculate processing rate (rows per second)
const uptimeSeconds = (Date.now() - this.startTime) / 1000;
const processingRate = this.dataProcessingStats.rowsProcessed / uptimeSeconds;
this.setGauge('processing_rate_rows_per_second', processingRate);
}
/**
* Record request metrics
*/
recordRequest(responseTime, success) {
this.requestCount++;
this.responseTimes.push(responseTime);
if (!success) {
this.errorCount++;
}
// Keep only recent response times
if (this.responseTimes.length > 1000) {
this.responseTimes = this.responseTimes.slice(-1000);
}
// Record as standard metrics
this.incrementCounter('requests_total', 1, { success: success.toString() });
this.recordHistogram('request_duration_ms', responseTime);
const errorRate = (this.errorCount / this.requestCount) * 100;
this.setGauge('error_rate_percent', errorRate);
}
/**
* Get current system metrics
*/
async getSystemMetrics() {
const memoryUsage = process.memoryUsage();
const uptimeMs = Date.now() - this.startTime;
const uptimeSeconds = uptimeMs / 1000;
// Calculate averages
const avgResponseTime = this.responseTimes.length > 0
? this.responseTimes.reduce((a, b) => a + b, 0) / this.responseTimes.length
: 0;
const p95ResponseTime = this.calculatePercentile(this.responseTimes, 0.95);
const p99ResponseTime = this.calculatePercentile(this.responseTimes, 0.99);
const requestsPerSecond = uptimeSeconds > 0 ? this.requestCount / uptimeSeconds : 0;
const errorRate = this.requestCount > 0 ? (this.errorCount / this.requestCount) * 100 : 0;
const processingRate = uptimeSeconds > 0 ? this.dataProcessingStats.rowsProcessed / uptimeSeconds : 0;
return {
timestamp: Date.now(),
system: {
uptime: uptimeMs,
memory: {
total: memoryUsage.heapTotal + memoryUsage.external,
used: memoryUsage.heapUsed,
free: memoryUsage.heapTotal - memoryUsage.heapUsed,
heapTotal: memoryUsage.heapTotal,
heapUsed: memoryUsage.heapUsed,
external: memoryUsage.external,
rss: memoryUsage.rss,
},
cpu: {
usage: process.cpuUsage().user / 1000000, // Convert to seconds
loadAverage: [],
},
process: {
pid: process.pid,
ppid: process.ppid || 0,
platform: process.platform,
arch: process.arch,
version: process.version,
},
},
application: {
requestsTotal: this.requestCount,
requestsPerSecond,
responseTimeMs: {
avg: avgResponseTime,
p95: p95ResponseTime,
p99: p99ResponseTime,
},
errorsTotal: this.errorCount,
errorRate,
dataProcessing: {
rowsProcessed: this.dataProcessingStats.rowsProcessed,
filesProcessed: this.dataProcessingStats.filesProcessed,
bytesProcessed: this.dataProcessingStats.bytesProcessed,
processingRate,
},
sections: {
section1: this.sectionMetrics.get('section1') || this.getDefaultSectionMetrics(),
section2: this.sectionMetrics.get('section2') || this.getDefaultSectionMetrics(),
section3: this.sectionMetrics.get('section3') || this.getDefaultSectionMetrics(),
section4: this.sectionMetrics.get('section4') || this.getDefaultSectionMetrics(),
section5: this.sectionMetrics.get('section5') || this.getDefaultSectionMetrics(),
section6: this.sectionMetrics.get('section6') || this.getDefaultSectionMetrics(),
},
},
};
}
/**
* Get metric summary with aggregations
*/
getMetricSummary(name) {
const timeSeries = this.metrics.get(name);
const config = this.configs.get(name);
if (!timeSeries || !config) {
return undefined;
}
const values = timeSeries.values.map((v) => v.value);
if (values.length === 0) {
return undefined;
}
const sortedValues = [...values].sort((a, b) => a - b);
const sum = values.reduce((a, b) => a + b, 0);
const avg = sum / values.length;
return {
name,
type: config.type,
description: config.description,
unit: config.unit,
currentValue: values[values.length - 1],
aggregations: {
min: Math.min(...values),
max: Math.max(...values),
avg,
p50: this.calculatePercentile(sortedValues, 0.5),
p95: this.calculatePercentile(sortedValues, 0.95),
p99: this.calculatePercentile(sortedValues, 0.99),
sum,
count: values.length,
},
lastUpdated: timeSeries.values[timeSeries.values.length - 1]?.timestamp || 0,
};
}
/**
* Get all metric summaries
*/
getAllMetricSummaries() {
return Array.from(this.configs.keys())
.map((name) => this.getMetricSummary(name))
.filter((summary) => summary !== undefined);
}
/**
* Export metrics in Prometheus format
*/
exportPrometheusMetrics() {
const lines = [];
const timestamp = Date.now();
for (const [name, config] of this.configs.entries()) {
const summary = this.getMetricSummary(name);
if (!summary)
continue;
lines.push(`# HELP datapilot_${name} ${config.description}`);
lines.push(`# TYPE datapilot_${name} ${config.type}`);
switch (config.type) {
case MetricType.COUNTER:
case MetricType.GAUGE:
lines.push(`datapilot_${name} ${summary.currentValue} ${timestamp}`);
break;
case MetricType.HISTOGRAM:
// Export histogram buckets and summary statistics
lines.push(`datapilot_${name}_count ${summary.aggregations.count} ${timestamp}`);
lines.push(`datapilot_${name}_sum ${summary.aggregations.sum} ${timestamp}`);
lines.push(`datapilot_${name}_avg ${summary.aggregations.avg} ${timestamp}`);
lines.push(`datapilot_${name}_p95 ${summary.aggregations.p95} ${timestamp}`);
lines.push(`datapilot_${name}_p99 ${summary.aggregations.p99} ${timestamp}`);
break;
case MetricType.SUMMARY:
lines.push(`datapilot_${name}_count ${summary.aggregations.count} ${timestamp}`);
lines.push(`datapilot_${name}_sum ${summary.aggregations.sum} ${timestamp}`);
lines.push(`datapilot_${name}{quantile="0.5"} ${summary.aggregations.p50} ${timestamp}`);
lines.push(`datapilot_${name}{quantile="0.95"} ${summary.aggregations.p95} ${timestamp}`);
lines.push(`datapilot_${name}{quantile="0.99"} ${summary.aggregations.p99} ${timestamp}`);
break;
}
}
return lines.join('\n') + '\n';
}
/**
* Export metrics in JSON format
*/
exportJSONMetrics() {
return {
timestamp: Date.now(),
uptime: Date.now() - this.startTime,
metrics: this.getAllMetricSummaries(),
system: {
requests: {
total: this.requestCount,
errors: this.errorCount,
errorRate: this.requestCount > 0 ? (this.errorCount / this.requestCount) * 100 : 0,
},
dataProcessing: this.dataProcessingStats,
sections: Object.fromEntries(this.sectionMetrics.entries()),
},
};
}
/**
* Start automatic metrics collection
*/
startCollection(intervalMs = 30000) {
if (this.isCollecting) {
logger_1.logger.warn('Metrics collection already started');
return;
}
this.isCollecting = true;
logger_1.logger.info(`Starting metrics collection with ${intervalMs}ms interval`);
// Collect system metrics periodically
this.collectionInterval = setInterval(async () => {
try {
await this.collectSystemMetrics();
}
catch (error) {
logger_1.logger.error(`Error collecting system metrics: ${error instanceof Error ? error.message : 'Unknown error'}`, {
operation: 'collectSystemMetrics',
error: error instanceof Error ? error.stack : String(error),
});
}
}, intervalMs);
// Aggregate metrics less frequently
this.aggregationInterval = setInterval(() => {
try {
this.aggregateMetrics();
}
catch (error) {
logger_1.logger.error(`Error aggregating metrics: ${error instanceof Error ? error.message : 'Unknown error'}`, {
operation: 'aggregateMetrics',
error: error instanceof Error ? error.stack : String(error),
});
}
}, intervalMs * 2);
}
/**
* Stop metrics collection
*/
stopCollection() {
if (!this.isCollecting) {
return;
}
this.isCollecting = false;
if (this.collectionInterval) {
clearInterval(this.collectionInterval);
this.collectionInterval = undefined;
}
if (this.aggregationInterval) {
clearInterval(this.aggregationInterval);
this.aggregationInterval = undefined;
}
logger_1.logger.info('Metrics collection stopped');
}
/**
* Reset all metrics
*/
reset() {
this.metrics.clear();
this.aggregations.clear();
this.requestCount = 0;
this.errorCount = 0;
this.responseTimes = [];
this.dataProcessingStats = {
rowsProcessed: 0,
filesProcessed: 0,
bytesProcessed: 0,
};
this.sectionMetrics.clear();
this.startTime = Date.now();
this.initializeSectionMetrics();
logger_1.logger.info('Metrics reset');
}
/**
* Dispose of the metrics collector
*/
dispose() {
this.stopCollection();
this.removeAllListeners();
this.metrics.clear();
this.configs.clear();
this.aggregations.clear();
}
async collectSystemMetrics() {
// Collect memory metrics
const memoryUsage = memory_manager_1.globalMemoryManager.checkMemoryUsage();
this.setGauge('memory_heap_used_bytes', memoryUsage.heapUsed);
this.setGauge('memory_heap_total_bytes', memoryUsage.heapTotal);
this.setGauge('memory_external_bytes', memoryUsage.external);
this.setGauge('memory_rss_bytes', memoryUsage.rss);
// Collect performance metrics
const performanceMonitor = (0, performance_monitor_1.getPerformanceMonitor)();
const performanceSummary = performanceMonitor.getPerformanceSummary();
if (performanceSummary.currentMetrics) {
this.setGauge('cpu_usage_seconds', performanceSummary.currentMetrics.cpuTime || 0);
this.setGauge('processing_rate_rows_per_second', performanceSummary.currentMetrics.processingRate || 0);
this.setGauge('throughput_operations_per_second', performanceSummary.currentMetrics.throughput || 0);
}
// Collect error metrics
const errorStats = error_handler_1.globalErrorHandler.getStats();
this.setGauge('errors_total', errorStats.totalErrors);
this.setGauge('critical_errors_total', errorStats.criticalErrors);
this.setGauge('recovered_errors_total', errorStats.recoveredErrors);
this.emit('systemMetricsCollected');
}
aggregateMetrics() {
for (const [name, config] of this.configs.entries()) {
const summary = this.getMetricSummary(name);
if (summary) {
this.aggregations.set(name, summary);
}
}
this.emit('metricsAggregated');
}
cleanupTimeSeries(timeSeries) {
const cutoff = Date.now() - timeSeries.retention;
timeSeries.values = timeSeries.values.filter((value) => value.timestamp > cutoff);
}
calculatePercentile(sortedValues, percentile) {
if (sortedValues.length === 0)
return 0;
const index = percentile * (sortedValues.length - 1);
const lower = Math.floor(index);
const upper = Math.ceil(index);
if (lower === upper) {
return sortedValues[lower];
}
const weight = index - lower;
return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
}
getDefaultSectionMetrics() {
return {
executionCount: 0,
totalDuration: 0,
avgDuration: 0,
maxDuration: 0,
errorCount: 0,
successRate: 100,
lastExecution: 0,
};
}
initializeDefaultMetrics() {
// Counter metrics
this.registerMetric({
name: 'requests_total',
type: MetricType.COUNTER,
description: 'Total number of requests processed',
unit: 'requests',
labels: ['success'],
});
this.registerMetric({
name: 'rows_processed_total',
type: MetricType.COUNTER,
description: 'Total number of data rows processed',
unit: 'rows',
});
this.registerMetric({
name: 'bytes_processed_total',
type: MetricType.COUNTER,
description: 'Total number of bytes processed',
unit: 'bytes',
});
this.registerMetric({
name: 'files_processed_total',
type: MetricType.COUNTER,
description: 'Total number of files processed',
unit: 'files',
});
this.registerMetric({
name: 'section_executions_total',
type: MetricType.COUNTER,
description: 'Total number of section executions',
unit: 'executions',
labels: ['section', 'success'],
});
// Gauge metrics
this.registerMetric({
name: 'memory_heap_used_bytes',
type: MetricType.GAUGE,
description: 'Current heap memory usage in bytes',
unit: 'bytes',
});
this.registerMetric({
name: 'memory_heap_total_bytes',
type: MetricType.GAUGE,
description: 'Total heap memory available in bytes',
unit: 'bytes',
});
this.registerMetric({
name: 'memory_external_bytes',
type: MetricType.GAUGE,
description: 'External memory usage in bytes',
unit: 'bytes',
});
this.registerMetric({
name: 'memory_rss_bytes',
type: MetricType.GAUGE,
description: 'Resident set size in bytes',
unit: 'bytes',
});
this.registerMetric({
name: 'cpu_usage_seconds',
type: MetricType.GAUGE,
description: 'CPU usage in seconds',
unit: 'seconds',
});
this.registerMetric({
name: 'processing_rate_rows_per_second',
type: MetricType.GAUGE,
description: 'Current data processing rate in rows per second',
unit: 'rows/second',
});
this.registerMetric({
name: 'throughput_operations_per_second',
type: MetricType.GAUGE,
description: 'Current operation throughput per second',
unit: 'operations/second',
});
this.registerMetric({
name: 'error_rate_percent',
type: MetricType.GAUGE,
description: 'Current error rate as percentage',
unit: 'percent',
});
this.registerMetric({
name: 'errors_total',
type: MetricType.GAUGE,
description: 'Total number of errors encountered',
unit: 'errors',
});
this.registerMetric({
name: 'critical_errors_total',
type: MetricType.GAUGE,
description: 'Total number of critical errors',
unit: 'errors',
});
this.registerMetric({
name: 'recovered_errors_total',
type: MetricType.GAUGE,
description: 'Total number of recovered errors',
unit: 'errors',
});
// Histogram metrics
this.registerMetric({
name: 'request_duration_ms',
type: MetricType.HISTOGRAM,
description: 'Request duration in milliseconds',
unit: 'milliseconds',
buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000],
});
this.registerMetric({
name: 'section_duration_ms',
type: MetricType.HISTOGRAM,
description: 'Section execution duration in milliseconds',
unit: 'milliseconds',
labels: ['section'],
buckets: [10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000],
});
logger_1.logger.info('Default metrics registered', { operation: 'initializeDefaultMetrics' });
}
initializeSectionMetrics() {
const sections = ['section1', 'section2', 'section3', 'section4', 'section5', 'section6'];
sections.forEach((section) => {
this.sectionMetrics.set(section, this.getDefaultSectionMetrics());
});
}
}
exports.MetricsCollector = MetricsCollector;
/**
* Global metrics collector instance
*/
exports.globalMetricsCollector = MetricsCollector.getInstance();
/**
* Metrics collection utilities
*/
class MetricsUtils {
/**
* Create a timing decorator for automatic duration tracking
*/
static timing(metricName, labels) {
return function (target, propertyName, descriptor) {
const method = descriptor.value;
descriptor.value = async function (...args) {
const endTimer = exports.globalMetricsCollector.startTimer(metricName);
try {
const result = await method.apply(this, args);
endTimer();
return result;
}
catch (error) {
endTimer();
throw error;
}
};
};
}
/**
* Create a counter decorator for automatic counting
*/
static counter(metricName, labels) {
return function (target, propertyName, descriptor) {
const method = descriptor.value;
descriptor.value = async function (...args) {
try {
const result = await method.apply(this, args);
exports.globalMetricsCollector.incrementCounter(metricName, 1, { ...labels, success: 'true' });
return result;
}
catch (error) {
exports.globalMetricsCollector.incrementCounter(metricName, 1, { ...labels, success: 'false' });
throw error;
}
};
};
}
/**
* Create metrics for data processing operations
*/
static recordDataOperation(operationType, rowCount, byteCount, duration, success) {
exports.globalMetricsCollector.recordDataProcessing(rowCount, byteCount, operationType === 'file');
exports.globalMetricsCollector.recordHistogram('data_operation_duration_ms', duration, {
operation: operationType,
success: success.toString(),
});
exports.globalMetricsCollector.incrementCounter('data_operations_total', 1, {
operation: operationType,
success: success.toString(),
});
}
}
exports.MetricsUtils = MetricsUtils;
//# sourceMappingURL=metrics-collector.js.map