datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
559 lines • 23.6 kB
JavaScript
"use strict";
/**
* Performance-Optimized Visualization Pipeline
*
* Advanced engine that optimizes visualization performance based on:
* - Data size and complexity analysis
* - Memory usage estimation and optimization
* - Adaptive sampling and aggregation strategies
* - Rendering performance optimization
* - Progressive loading and streaming techniques
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.PerformanceOptimizer = void 0;
/**
* Performance Optimization Engine
*/
class PerformanceOptimizer {
/**
* Analyze data characteristics and generate performance profile
*/
static analyzePerformance(rowCount, columnCount, chartType, interactionComplexity = 1) {
const dataSize = this.categorizeDataSize(rowCount, columnCount);
const complexity = this.assessComplexity(rowCount, columnCount, chartType, interactionComplexity);
const memoryEstimate = this.estimateMemoryUsage(dataSize, complexity, chartType);
const renderingStrategy = this.selectRenderingStrategy(dataSize, complexity, chartType);
const optimizations = this.generateOptimizationRecommendations(dataSize, complexity, chartType);
const adaptiveStrategies = this.designAdaptiveStrategies(dataSize, complexity);
const performanceMetrics = this.calculatePerformanceMetrics(dataSize, complexity, renderingStrategy);
return {
dataSize,
complexity,
memoryEstimate,
renderingStrategy,
optimizationRecommendations: optimizations,
adaptiveStrategies,
performanceMetrics,
};
}
/**
* Generate intelligent sampling configuration for large datasets
*/
static generateSamplingStrategy(rowCount, columnCount, chartType, preserveStatistics = true) {
if (rowCount <= 1000) {
return {
technique: 'random',
sampleSize: rowCount,
preserveOutliers: true,
maintainDistribution: true,
reasoning: 'Small dataset - no sampling required',
quality: 'high',
};
}
let technique;
let sampleSize;
let quality;
// Determine optimal sampling based on data size and chart type
if (rowCount > 1000000) {
// Massive datasets require aggressive sampling
technique = 'adaptive';
sampleSize = Math.min(10000, Math.max(1000, Math.sqrt(rowCount)));
quality = 'medium';
}
else if (rowCount > 100000) {
// Large datasets benefit from stratified sampling
technique = 'stratified';
sampleSize = Math.min(5000, rowCount * 0.1);
quality = 'high';
}
else if (rowCount > 10000) {
// Medium datasets use systematic sampling
technique = 'systematic';
sampleSize = Math.min(2000, rowCount * 0.2);
quality = 'high';
}
else {
// Smaller large datasets use random sampling
technique = 'random';
sampleSize = Math.min(1000, rowCount * 0.5);
quality = 'high';
}
// Adjust for chart-specific requirements
if (chartType.includes('scatter') || chartType.includes('density')) {
sampleSize *= 1.5; // Scatter plots benefit from more points
}
else if (chartType.includes('histogram') || chartType.includes('binned')) {
sampleSize *= 0.8; // Aggregated charts need fewer points
}
return {
technique,
sampleSize: Math.floor(sampleSize),
preserveOutliers: preserveStatistics,
maintainDistribution: preserveStatistics,
reasoning: `${technique} sampling selected for ${rowCount} rows to balance performance and quality`,
quality,
};
}
/**
* Design aggregation strategy for performance optimization
*/
static designAggregationStrategy(dataSize, chartType, targetPoints = 1000) {
const reductionNeeded = dataSize.totalDataPoints / targetPoints;
if (reductionNeeded <= 1) {
return {
method: 'downsampling',
parameters: { factor: 1 },
dataReduction: 0,
informationRetention: 100,
visualFidelity: 100,
};
}
let method;
let informationRetention;
let visualFidelity;
if (chartType.includes('histogram') || chartType.includes('density')) {
method = 'binning';
informationRetention = 95;
visualFidelity = 98;
}
else if (chartType.includes('scatter') && reductionNeeded > 10) {
method = 'clustering';
informationRetention = 85;
visualFidelity = 90;
}
else if (chartType.includes('line') || chartType.includes('time')) {
method = 'summarization';
informationRetention = 90;
visualFidelity = 95;
}
else {
method = 'downsampling';
informationRetention = 80;
visualFidelity = 85;
}
const dataReduction = Math.min(95, (1 - 1 / reductionNeeded) * 100);
return {
method,
parameters: this.generateAggregationParameters(method, reductionNeeded, chartType),
dataReduction,
informationRetention,
visualFidelity,
};
}
/**
* Create progressive loading plan for complex visualizations
*/
static createProgressiveLoadingPlan(dataSize, complexity, chartType) {
const stages = [];
if (dataSize.category === 'tiny' || dataSize.category === 'small') {
// Simple loading for small datasets
stages.push({
stage: 1,
description: 'Load complete visualization',
dataVolume: 100,
estimatedTime: 500,
visualElements: ['all chart elements', 'full interactivity'],
fallbackOptions: ['static image fallback'],
});
}
else {
// Progressive loading for larger datasets
stages.push({
stage: 1,
description: 'Load basic chart structure and sample data',
dataVolume: 10,
estimatedTime: 200,
visualElements: ['axes', 'basic layout', 'sample points'],
fallbackOptions: ['loading indicator', 'skeleton chart'],
});
stages.push({
stage: 2,
description: 'Load aggregated data representation',
dataVolume: 50,
estimatedTime: 800,
visualElements: ['aggregated data', 'basic interactions'],
fallbackOptions: ['static aggregated view'],
});
if (complexity.level === 'complex' || complexity.level === 'extreme') {
stages.push({
stage: 3,
description: 'Load full detail on demand',
dataVolume: 100,
estimatedTime: 2000,
visualElements: ['full dataset', 'advanced interactions', 'annotations'],
fallbackOptions: ['zoom-to-detail', 'on-demand loading'],
});
}
}
return {
stages,
initialDisplay: 'Basic chart structure with loading indicator',
progressIndicators: ['Progress bar', 'Loading animation', 'Data count'],
userExperience: this.describeUserExperience(stages, complexity),
};
}
// Helper methods for categorization and analysis
static categorizeDataSize(rowCount, columnCount) {
const totalDataPoints = rowCount * columnCount;
let category;
let characterization;
let scalingChallenges;
if (totalDataPoints < 1000) {
category = 'tiny';
characterization = 'Minimal dataset suitable for any visualization technique';
scalingChallenges = [];
}
else if (totalDataPoints < 10000) {
category = 'small';
characterization = 'Small dataset with excellent performance for all chart types';
scalingChallenges = ['Minor rendering considerations for complex interactions'];
}
else if (totalDataPoints < 100000) {
category = 'medium';
characterization = 'Medium dataset requiring moderate optimization';
scalingChallenges = ['Canvas vs SVG considerations', 'Interaction latency management'];
}
else if (totalDataPoints < 1000000) {
category = 'large';
characterization = 'Large dataset requiring significant optimization strategies';
scalingChallenges = ['Memory management', 'Render performance', 'Progressive loading'];
}
else if (totalDataPoints < 10000000) {
category = 'huge';
characterization = 'Huge dataset requiring advanced optimization and aggregation';
scalingChallenges = [
'Aggressive sampling',
'Streaming data',
'Client-side processing limits',
];
}
else {
category = 'massive';
characterization = 'Massive dataset requiring specialized big data visualization techniques';
scalingChallenges = [
'Server-side aggregation',
'Streaming protocols',
'Progressive enhancement',
];
}
return {
category,
rowCount,
columnCount,
totalDataPoints,
characterization,
scalingChallenges,
};
}
static assessComplexity(rowCount, columnCount, chartType, interactionComplexity) {
const factors = [];
let computationalLoad = 1;
let renderingLoad = 1;
// Data size complexity
if (rowCount > 100000) {
factors.push({
factor: 'Large dataset',
impact: 'high',
description: `${rowCount} rows require optimization`,
mitigation: 'Sampling, aggregation, progressive loading',
});
computationalLoad *= 2;
renderingLoad *= 1.5;
}
// Chart type complexity
if (chartType.includes('density') || chartType.includes('contour')) {
factors.push({
factor: 'Complex chart type',
impact: 'medium',
description: 'Density/contour charts require advanced calculations',
mitigation: 'Pre-computed binning, WebGL acceleration',
});
computationalLoad *= 1.5;
renderingLoad *= 1.3;
}
// Multi-dimensional complexity
if (columnCount > 5) {
factors.push({
factor: 'High dimensionality',
impact: 'medium',
description: `${columnCount} columns create encoding complexity`,
mitigation: 'Dimensionality reduction, progressive disclosure',
});
computationalLoad *= 1.2;
}
// Interaction complexity
if (interactionComplexity > 2) {
factors.push({
factor: 'Complex interactions',
impact: 'medium',
description: 'Multiple interaction types increase complexity',
mitigation: 'Debounced events, efficient event handling',
});
renderingLoad *= 1.4;
}
const totalComplexity = computationalLoad * renderingLoad;
let level;
if (totalComplexity < 2)
level = 'simple';
else if (totalComplexity < 4)
level = 'moderate';
else if (totalComplexity < 8)
level = 'complex';
else
level = 'extreme';
return {
level,
factors,
computationalLoad,
renderingLoad,
interactionComplexity,
};
}
static estimateMemoryUsage(dataSize, complexity, chartType) {
// Base data memory (8 bytes per number, assuming mixed data types)
const baseDataMemory = (dataSize.totalDataPoints * 6) / (1024 * 1024); // MB
// Rendering memory depends on chart type and complexity
let renderingMultiplier = 1;
if (chartType.includes('canvas'))
renderingMultiplier = 1.5;
if (chartType.includes('webgl'))
renderingMultiplier = 2;
if (complexity.level === 'complex')
renderingMultiplier *= 1.3;
const renderingMemory = baseDataMemory * renderingMultiplier;
// Interaction memory for event handling and state
const interactionMemory = Math.min(50, baseDataMemory * 0.2);
const totalEstimate = baseDataMemory + renderingMemory + interactionMemory;
return {
baseDataMemory,
renderingMemory,
interactionMemory,
totalEstimate,
memoryProfile: {
domElements: renderingMemory * 0.4,
canvasMemory: renderingMemory * 0.3,
dataStructures: baseDataMemory * 0.8,
caching: Math.min(20, totalEstimate * 0.1),
buffers: renderingMemory * 0.2,
},
optimizationPotential: Math.min(80, Math.max(10, totalEstimate * 0.3)),
};
}
static selectRenderingStrategy(dataSize, complexity, chartType) {
let technique;
let reasoning;
let expectedPerformance;
let limitations = [];
if (dataSize.totalDataPoints > 50000 || complexity.level === 'extreme') {
technique = 'webgl';
reasoning = 'Large dataset and high complexity require GPU acceleration';
expectedPerformance = 'Excellent for large datasets, 60fps animations possible';
limitations = ['WebGL compatibility required', 'More complex implementation'];
}
else if (dataSize.totalDataPoints > 5000 || chartType.includes('canvas')) {
technique = 'canvas';
reasoning = 'Medium to large dataset benefits from raster rendering';
expectedPerformance = 'Good performance, handles thousands of elements well';
limitations = ['No DOM manipulation', 'Custom event handling required'];
}
else if (dataSize.category === 'small' || dataSize.category === 'tiny') {
technique = 'svg';
reasoning = 'Small dataset allows for DOM-based vector rendering';
expectedPerformance = 'Excellent quality, perfect scalability, easy interactions';
limitations = ['Performance degrades with >1000 elements', 'Memory intensive'];
}
else {
technique = 'hybrid';
reasoning = 'Mixed approach balances quality and performance';
expectedPerformance = 'Good balance of quality and performance';
limitations = ['More complex implementation', 'Careful optimization required'];
}
const optimizations = this.generateRenderingOptimizations(technique, dataSize, complexity);
return {
technique,
reasoning,
expectedPerformance,
limitations,
optimizations,
};
}
static generateRenderingOptimizations(technique, dataSize, complexity) {
const optimizations = [];
if (dataSize.totalDataPoints > 10000) {
optimizations.push({
technique: 'Level of Detail (LOD)',
description: 'Reduce visual complexity at different zoom levels',
impact: 'high',
implementation: 'Progressive simplification based on viewport scale',
tradeoffs: ['Complexity in implementation', 'Visual detail at distance'],
});
}
if (technique === 'canvas' || technique === 'webgl') {
optimizations.push({
technique: 'Viewport Culling',
description: 'Only render elements visible in current viewport',
impact: 'high',
implementation: 'Spatial indexing with efficient visibility testing',
tradeoffs: ['Additional spatial calculations', 'Memory for spatial index'],
});
}
if (complexity.interactionComplexity > 1) {
optimizations.push({
technique: 'Event Debouncing',
description: 'Throttle expensive interaction calculations',
impact: 'medium',
implementation: 'RequestAnimationFrame-based event handling',
tradeoffs: ['Slight interaction latency', 'More complex event logic'],
});
}
return optimizations;
}
static generateOptimizationRecommendations(dataSize, complexity, chartType) {
const recommendations = [];
// Data optimization
if (dataSize.totalDataPoints > 50000) {
recommendations.push({
category: 'data',
priority: 'critical',
technique: 'Intelligent Sampling',
description: 'Reduce dataset size while preserving statistical properties',
expectedImprovement: '70-90% performance improvement',
implementationComplexity: 'moderate',
codeExample: 'const sampled = stratifiedSample(data, 5000, preserveOutliers: true);',
});
}
// Rendering optimization
if (complexity.renderingLoad > 2) {
recommendations.push({
category: 'rendering',
priority: 'high',
technique: 'Progressive Enhancement',
description: 'Load visualization in stages for better perceived performance',
expectedImprovement: '50-80% faster initial load',
implementationComplexity: 'complex',
});
}
// Memory optimization
if (dataSize.category === 'large' || dataSize.category === 'huge') {
recommendations.push({
category: 'memory',
priority: 'high',
technique: 'Data Streaming',
description: 'Process data in chunks to avoid memory spikes',
expectedImprovement: '60% reduction in peak memory usage',
implementationComplexity: 'complex',
});
}
return recommendations;
}
static designAdaptiveStrategies(dataSize, complexity) {
const strategies = [];
if (dataSize.totalDataPoints > 10000) {
strategies.push({
strategy: 'Dynamic Level of Detail',
trigger: 'Zoom level changes or performance degradation detected',
implementation: 'Automatically adjust point density based on viewport scale',
fallbackOptions: ['Fixed low detail', 'Progressive loading', 'Static overview'],
performanceGain: 60,
});
}
if (complexity.level === 'complex' || complexity.level === 'extreme') {
strategies.push({
strategy: 'Graceful Degradation',
trigger: 'Frame rate drops below 30fps for 2 seconds',
implementation: 'Progressively disable expensive features (animations, smooth transitions)',
fallbackOptions: ['Static visualization', 'Simplified interactions', 'Pre-rendered images'],
performanceGain: 40,
});
}
return strategies;
}
static calculatePerformanceMetrics(dataSize, complexity, renderingStrategy) {
// Empirical formulas based on data size and complexity
const baseRenderTime = Math.log10(dataSize.totalDataPoints) * 100;
const complexityMultiplier = Math.pow(complexity.computationalLoad, 0.8);
let renderingMultiplier = 1;
switch (renderingStrategy.technique) {
case 'svg':
renderingMultiplier = 1.5;
break;
case 'canvas':
renderingMultiplier = 1.0;
break;
case 'webgl':
renderingMultiplier = 0.3;
break;
case 'hybrid':
renderingMultiplier = 0.8;
break;
}
const estimatedRenderTime = baseRenderTime * complexityMultiplier * renderingMultiplier;
const estimatedMemoryPeak = Math.max(10, dataSize.totalDataPoints * 0.001);
const userInteractionLatency = Math.min(500, estimatedRenderTime * 0.1);
const timeToFirstPaint = estimatedRenderTime * 0.6;
let frameRate = 60;
if (complexity.level === 'extreme')
frameRate = 30;
else if (complexity.level === 'complex')
frameRate = 45;
const scalabilityScore = Math.max(0, Math.min(100, 100 - (dataSize.totalDataPoints / 1000000) * 50 - complexity.computationalLoad * 10));
return {
estimatedRenderTime,
estimatedMemoryPeak,
userInteractionLatency,
timeToFirstPaint,
frameRate,
scalabilityScore,
};
}
static generateAggregationParameters(method, reductionFactor, chartType) {
switch (method) {
case 'binning':
return {
binCount: Math.min(100, Math.max(10, Math.sqrt(reductionFactor) * 10)),
adaptive: true,
preserveOutliers: true,
};
case 'clustering':
return {
clusters: Math.min(1000, Math.max(50, 1000 / Math.sqrt(reductionFactor))),
algorithm: 'kmeans',
preserveShape: true,
};
case 'downsampling':
return {
factor: Math.ceil(reductionFactor),
method: 'systematic',
preserveEdges: true,
};
case 'summarization':
return {
windowSize: Math.ceil(reductionFactor),
aggregation: 'mean',
preservePeaks: true,
};
default:
// Default aggregation for unknown methods
return {
level: Math.min(10, Math.max(2, Math.ceil(reductionFactor / 100))),
method: 'adaptive',
fallbackStrategy: 'statistical_sampling',
preserveDistribution: true,
qualityThreshold: 0.85,
};
}
}
static describeUserExperience(stages, complexity) {
if (stages.length === 1) {
return 'Instant visualization load with full functionality immediately available';
}
else if (stages.length === 2) {
return 'Quick initial display followed by enhanced detail loading in background';
}
else {
return 'Progressive enhancement: basic chart appears instantly, full details load on-demand for optimal responsiveness';
}
}
}
exports.PerformanceOptimizer = PerformanceOptimizer;
//# sourceMappingURL=performance-optimizer.js.map