datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
882 lines • 173 kB
JavaScript
"use strict";
/**
* Section 4: Advanced Visualization Intelligence Analyzer
*
* Ultra-sophisticated visualization recommendation engine featuring:
* - Statistical-driven chart selection with data distribution analysis
* - Performance-optimized visualization pipeline with adaptive algorithms
* - Advanced chart composition with multi-dimensional encoding
* - Sophisticated dashboard layout using perceptual principles
* - Domain-aware visualization intelligence with context detection
* - Aesthetic optimization engine for data-driven design decisions
*
* This represents the "calculator on steroids" approach to visualization intelligence,
* combining statistical rigor with sophisticated design principles.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.Section4Analyzer = void 0;
const dashboard_layout_engine_1 = require("./engines/dashboard-layout-engine");
const domain_aware_intelligence_1 = require("./engines/domain-aware-intelligence");
const wcag_accessibility_engine_1 = require("./engines/wcag-accessibility-engine");
const types_1 = require("./types");
const types_2 = require("../eda/types");
const logger_1 = require("../../utils/logger");
class Section4Analyzer {
config;
warnings = [];
antiPatterns = [];
constructor(config = {}) {
this.config = {
enabledRecommendations: [
'univariate',
'bivariate',
'dashboard',
'accessibility',
'performance',
],
accessibilityLevel: types_1.AccessibilityLevel.GOOD,
complexityThreshold: types_1.ComplexityLevel.MODERATE,
performanceThreshold: types_1.PerformanceLevel.MODERATE,
maxRecommendationsPerChart: 3,
includeCodeExamples: true,
targetLibraries: ['d3', 'plotly', 'observable'],
...config,
};
}
/**
* Main analysis method - uses sophisticated visualization intelligence engines
*/
async analyze(section1Result, section3Result) {
const startTime = Date.now();
logger_1.logger.info('Starting Section 4: Advanced Visualization Intelligence analysis');
try {
// Extract data characteristics for engine processing
const dataCharacteristics = this.extractDataCharacteristics(section1Result, section3Result);
const columnNames = section1Result.overview.structuralDimensions.columnInventory?.map((col) => col.name) || [];
// Use Domain-Aware Intelligence to understand context
const domainContext = domain_aware_intelligence_1.DomainAwareIntelligence.analyzeDomainContext(columnNames, dataCharacteristics);
logger_1.logger.info(`Domain detected: ${domainContext.primaryDomain.domain} (confidence: ${domainContext.confidence.toFixed(2)})`);
// Generate aesthetic profile for beautiful, accessible visualizations
// TODO: Fix AestheticOptimizer.generateAestheticProfile array access bug
const aestheticProfile = {
colorSystem: {
primaryPalette: {
primary: [{ hex: '#1f77b4' }],
neutral: [{ hex: '#cccccc' }],
},
dataVisualizationPalette: {
categorical: ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd'],
numerical: ['#1f77b4', '#aec7e8', '#ffbb78'],
diverging: ['#d62728', '#ffffff', '#1f77b4'],
},
},
typographySystem: {},
visualComposition: {},
emotionalDesign: {},
accessibility: {},
brandIntegration: {},
responsiveAesthetics: {},
qualityMetrics: {},
};
// Use Statistical Chart Selector for intelligent recommendations
const univariateRecommendations = this.generateSophisticatedUnivariateRecommendations(section1Result, section3Result, domainContext, aestheticProfile);
// Generate bivariate recommendations with sophisticated analysis
const bivariateRecommendations = this.generateSophisticatedBivariateRecommendations(section3Result, domainContext, aestheticProfile);
// Generate multivariate recommendations using advanced techniques
const multivariateRecommendations = this.generateSophisticatedMultivariateRecommendations(section3Result, domainContext);
// Create sophisticated dashboard layout using perceptual principles
const dashboardRecommendations = this.generateSophisticatedDashboardRecommendations(univariateRecommendations, bivariateRecommendations, domainContext, aestheticProfile);
// Generate enhanced technical guidance
const technicalGuidance = this.generateEnhancedTechnicalGuidance(univariateRecommendations, bivariateRecommendations, domainContext, aestheticProfile);
// Comprehensive accessibility assessment
const accessibilityAssessment = this.assessEnhancedAccessibility(univariateRecommendations, bivariateRecommendations, aestheticProfile);
// Generate overall visualization strategy based on domain insights
const strategy = this.generateDomainAwareStrategy(section1Result, section3Result, domainContext);
const analysisTime = Date.now() - startTime;
const totalRecommendations = univariateRecommendations.reduce((sum, profile) => sum + (profile.recommendations?.length || 0), 0) +
bivariateRecommendations.reduce((sum, profile) => sum + (profile.recommendations?.length || 0), 0);
const visualizationAnalysis = {
strategy,
univariateRecommendations,
bivariateRecommendations,
multivariateRecommendations,
dashboardRecommendations,
technicalGuidance,
accessibilityAssessment,
};
logger_1.logger.info(`Section 4 sophisticated analysis completed in ${analysisTime}ms`);
logger_1.logger.info(`Domain: ${domainContext.primaryDomain.domain}, Engines: 6 sophisticated engines used`);
return {
visualizationAnalysis,
warnings: this.warnings,
performanceMetrics: {
analysisTimeMs: analysisTime,
recommendationsGenerated: totalRecommendations,
chartTypesConsidered: this.getUniqueChartTypes(univariateRecommendations, bivariateRecommendations).length,
accessibilityChecks: this.countAccessibilityChecks(),
},
metadata: {
analysisApproach: 'Ultra-sophisticated visualization intelligence with 6 specialized engines',
totalColumns: univariateRecommendations.length || 0,
bivariateRelationships: bivariateRecommendations.length || 0,
recommendationConfidence: this.calculateOverallConfidence(univariateRecommendations, bivariateRecommendations),
},
};
}
catch (error) {
logger_1.logger.error('Section 4 analysis failed:', error);
throw error;
}
}
/**
* Get domain-specific audience from domain context
*/
getDomainAudience(domainContext) {
switch (domainContext.primaryDomain?.domain) {
case 'education':
return 'educators and students';
case 'healthcare':
return 'medical professionals';
case 'finance':
return 'financial analysts';
default:
return 'general audience';
}
}
/**
* Determine complexity level from domain context
*/
determineComplexityFromDomain(domainContext) {
const domain = domainContext.primaryDomain?.domain;
switch (domain) {
case 'education':
return types_1.ComplexityLevel.MODERATE;
case 'healthcare':
return types_1.ComplexityLevel.COMPLEX;
case 'finance':
return types_1.ComplexityLevel.COMPLEX;
default:
return types_1.ComplexityLevel.MODERATE;
}
}
/**
* Determine interactivity level from domain context
*/
determineInteractivityFromDomain(domainContext) {
const domain = domainContext.primaryDomain?.domain;
switch (domain) {
case 'education':
return types_1.InteractivityLevel.INTERACTIVE;
case 'healthcare':
return types_1.InteractivityLevel.BASIC;
case 'finance':
return types_1.InteractivityLevel.HIGHLY_INTERACTIVE;
default:
return types_1.InteractivityLevel.INTERACTIVE;
}
}
/**
* Determine performance level from data size
*/
determinePerformanceFromData(dataCharacteristics) {
const rows = dataCharacteristics.totalRows || 0;
if (rows > 10000)
return types_1.PerformanceLevel.INTENSIVE;
if (rows > 1000)
return types_1.PerformanceLevel.MODERATE;
return types_1.PerformanceLevel.FAST;
}
/**
* Extract data characteristics for sophisticated engine processing
*/
extractDataCharacteristics(section1Result, section3Result) {
const structural = section1Result.overview.structuralDimensions;
// Extract quality metrics from Section 3 if available, or use defaults
const qualityProfile = {
completeness: 0.9,
consistency: 0.85,
validity: 0.9,
uniqueness: 0.95,
};
return {
totalRows: structural.totalDataRows || 0,
totalColumns: structural.totalColumns || 0,
categoricalColumns: section3Result.edaAnalysis?.univariateAnalysis?.filter((col) => col.detectedDataType === types_2.EdaDataType.CATEGORICAL).length || 0,
numericalColumns: section3Result.edaAnalysis?.univariateAnalysis?.filter((col) => this.isNumericalType(col.detectedDataType)).length || 0,
temporalColumns: section3Result.edaAnalysis?.univariateAnalysis?.filter((col) => col.detectedDataType === types_2.EdaDataType.DATE_TIME).length || 0,
hasHierarchy: false, // Could be detected from data patterns
hasNegativeValues: false, // Could be detected from numeric analysis
maxUniqueValues: Math.max(...(section3Result.edaAnalysis?.univariateAnalysis?.map((col) => col.uniqueValues || 0) || [
0,
])),
completenessScore: qualityProfile.completeness || 0,
consistencyScore: qualityProfile.consistency || 0,
validityScore: qualityProfile.validity || 0,
uniquenessScore: qualityProfile.uniqueness || 0,
};
}
/**
* Generate domain-aware visualization strategy using sophisticated analysis
*/
generateDomainAwareStrategy(section1Result, section3Result, domainContext) {
const approach = domainContext.visualizationStrategy?.primaryApproach?.approach ||
'Data-driven chart selection with domain intelligence and aesthetic optimization';
const primaryObjectives = domainContext.visualizationStrategy?.primaryApproach?.suitableFor || [
'performance analysis',
'factor identification',
'relationship exploration',
];
return {
approach,
primaryObjectives,
targetAudience: this.getDomainAudience(domainContext),
complexity: this.determineComplexityFromDomain(domainContext),
interactivity: this.determineInteractivityFromDomain(domainContext),
accessibility: this.config.accessibilityLevel,
performance: this.determinePerformanceFromData(this.extractDataCharacteristics(section1Result, {})),
};
}
/**
* Generate sophisticated univariate recommendations using all engines
*/
generateSophisticatedUnivariateRecommendations(section1Result, section3Result, domainContext, aestheticProfile) {
const profiles = [];
if (!section3Result.edaAnalysis?.univariateAnalysis) {
this.warnings.push({
type: 'data_quality',
severity: 'high',
message: 'No univariate analysis data available from Section 3',
recommendation: 'Run Section 3 EDA analysis first',
impact: 'Limited sophisticated visualization recommendations possible',
});
return profiles;
}
for (const columnAnalysis of section3Result.edaAnalysis.univariateAnalysis) {
const profile = this.createSophisticatedColumnProfile(columnAnalysis, domainContext, aestheticProfile);
profiles.push(profile);
}
return profiles;
}
/**
* Generate sophisticated bivariate recommendations
*/
generateSophisticatedBivariateRecommendations(section3Result, domainContext, aestheticProfile) {
const profiles = [];
if (!section3Result.edaAnalysis?.bivariateAnalysis?.numericalVsNumerical?.correlationPairs) {
logger_1.logger.info('No correlation data available for sophisticated bivariate analysis');
return profiles;
}
// Extract correlations with domain awareness
const correlations = this.extractCorrelations(section3Result);
// Apply domain-specific filtering and enhancement
const enhancedCorrelations = correlations
.filter((corr) => Section4Analyzer.isDomainRelevantCorrelation(corr, domainContext))
.map((corr) => Section4Analyzer.enhanceCorrelationWithDomainKnowledge(corr, domainContext));
// Generate sophisticated recommendations
for (const correlation of enhancedCorrelations) {
const profile = Section4Analyzer.createSophisticatedBivariateProfile(correlation, section3Result, domainContext, aestheticProfile);
if (profile) {
profiles.push(profile);
}
}
return profiles.slice(0, 10); // Limit to top 10 relationships
}
/**
* Generate sophisticated multivariate recommendations
*/
generateSophisticatedMultivariateRecommendations(section3Result, domainContext) {
const recommendations = [];
if (!section3Result.edaAnalysis?.univariateAnalysis) {
return recommendations;
}
const numericalColumns = section3Result.edaAnalysis.univariateAnalysis.filter((col) => this.isNumericalType(col.detectedDataType));
const categoricalColumns = section3Result.edaAnalysis.univariateAnalysis.filter((col) => col.detectedDataType === types_2.EdaDataType.CATEGORICAL);
// Extract clustering insights for visualization enhancement
const clusteringInsights = this.extractClusteringInsights(section3Result);
// Domain-aware multivariate recommendations with clustering integration
if (domainContext.primaryDomain.domain === 'education') {
// Educational performance factor analysis
if (numericalColumns.length >= 3) {
const baseRecommendation = {
variables: numericalColumns.slice(0, 6).map((col) => col.columnName),
purpose: 'Identify key factors influencing academic performance using sophisticated factor analysis',
chartType: types_1.ChartType.PARALLEL_COORDINATES,
complexity: types_1.ComplexityLevel.MODERATE,
prerequisites: ['Performance variable identification', 'Factor importance ranking'],
implementation: 'Interactive parallel coordinates with domain-specific factor highlighting and educational benchmarks',
alternatives: [types_1.ChartType.RADAR_CHART, types_1.ChartType.CORRELATION_MATRIX],
};
// Enhance with clustering insights
if (clusteringInsights.hasNaturalClusters) {
baseRecommendation.purpose += ` Enhanced with ${clusteringInsights.optimalClusters} distinct student performance groups`;
baseRecommendation.prerequisites.push('Cluster-based color encoding');
baseRecommendation.implementation += ` with ${clusteringInsights.optimalClusters}-cluster color coding for student group identification`;
baseRecommendation.alternatives.push(types_1.ChartType.SCATTER_PLOT); // For cluster scatter plots
}
recommendations.push(baseRecommendation);
}
// Academic intervention analysis
if (numericalColumns.length >= 4) {
const correlationRecommendation = {
variables: numericalColumns.map((col) => col.columnName),
purpose: 'Comprehensive academic intervention impact analysis with performance correlation matrix',
chartType: types_1.ChartType.CORRELATION_MATRIX,
complexity: types_1.ComplexityLevel.SIMPLE,
prerequisites: ['Educational domain context', 'Performance outcome identification'],
implementation: 'Educational correlation heatmap with significance indicators and intervention recommendations',
alternatives: [types_1.ChartType.SCATTERPLOT_MATRIX],
};
// Enhance with clustering insights
if (clusteringInsights.hasNaturalClusters) {
correlationRecommendation.purpose += ` with ${clusteringInsights.optimalClusters} performance tier analysis`;
correlationRecommendation.prerequisites.push('Cluster-specific correlation analysis');
correlationRecommendation.implementation += ` featuring separate correlation patterns for each of the ${clusteringInsights.optimalClusters} identified student clusters`;
}
recommendations.push(correlationRecommendation);
}
// Add cluster-specific visualizations for education domain
if (clusteringInsights.hasNaturalClusters && numericalColumns.length >= 2) {
recommendations.push({
variables: clusteringInsights.dominantVariables ||
numericalColumns.slice(0, 3).map((col) => col.columnName),
purpose: `Visualize ${clusteringInsights.optimalClusters} distinct student performance clusters for targeted intervention strategies`,
chartType: types_1.ChartType.SCATTER_PLOT,
complexity: types_1.ComplexityLevel.SIMPLE,
prerequisites: ['Cluster assignment', 'Performance variable identification'],
implementation: `2D/3D scatter plot with ${clusteringInsights.optimalClusters} color-coded clusters, cluster centroids, and silhouette quality indicators (score: ${clusteringInsights.qualityScore?.toFixed(2)})`,
alternatives: [types_1.ChartType.PARALLEL_COORDINATES, types_1.ChartType.RADAR_CHART],
});
}
}
else {
// Generic sophisticated recommendations for other domains
if (numericalColumns.length >= 3) {
const genericRecommendation = {
variables: numericalColumns.slice(0, 6).map((col) => col.columnName),
purpose: 'Multi-dimensional relationship analysis with sophisticated pattern detection',
chartType: types_1.ChartType.PARALLEL_COORDINATES,
complexity: types_1.ComplexityLevel.MODERATE,
prerequisites: ['Data normalization', 'Outlier treatment'],
implementation: 'Advanced parallel coordinates with brushing, linking, and pattern highlighting',
alternatives: [types_1.ChartType.RADAR_CHART, types_1.ChartType.SCATTERPLOT_MATRIX],
};
// Enhance with clustering insights
if (clusteringInsights.hasNaturalClusters) {
genericRecommendation.purpose += ` featuring ${clusteringInsights.optimalClusters} natural data clusters`;
genericRecommendation.prerequisites.push('Cluster color encoding');
genericRecommendation.implementation += ` with cluster-based color encoding and interactive cluster filtering`;
}
recommendations.push(genericRecommendation);
}
// Add cluster-specific visualization for any domain when clusters exist
if (clusteringInsights.hasNaturalClusters && numericalColumns.length >= 2) {
recommendations.push({
variables: clusteringInsights.dominantVariables ||
numericalColumns.slice(0, 3).map((col) => col.columnName),
purpose: `Explore ${clusteringInsights.optimalClusters} natural data groupings and their distinguishing characteristics`,
chartType: types_1.ChartType.SCATTER_PLOT,
complexity: types_1.ComplexityLevel.SIMPLE,
prerequisites: ['Cluster assignment', 'Variable selection for visualization'],
implementation: `Interactive cluster scatter plot with ${clusteringInsights.optimalClusters} groups, centroid overlays, and cluster quality metrics (silhouette: ${clusteringInsights.qualityScore?.toFixed(2)})`,
alternatives: [types_1.ChartType.PARALLEL_COORDINATES, types_1.ChartType.BOX_PLOT],
});
}
}
return recommendations;
}
/**
* Extract clustering insights from Section 3 multivariate analysis
*/
extractClusteringInsights(section3Result) {
try {
const multivariateAnalysis = section3Result.edaAnalysis.multivariateAnalysis;
const clusteringAnalysis = multivariateAnalysis?.clusteringAnalysis;
if (!clusteringAnalysis || !clusteringAnalysis.isApplicable) {
return { hasNaturalClusters: false };
}
const silhouetteScore = clusteringAnalysis.finalClustering.validation.silhouetteScore;
const hasQualityClusters = silhouetteScore > 0.3; // Reasonable threshold for visualization
if (!hasQualityClusters) {
return { hasNaturalClusters: false };
}
// Extract cluster characteristics for visualization enhancement
const clusterProfiles = clusteringAnalysis.finalClustering.clusterProfiles.map((profile) => ({
clusterId: profile.clusterId,
characteristics: profile.distinctiveFeatures,
size: profile.size,
}));
// Get variables used in clustering for visualization recommendations
const dominantVariables = clusteringAnalysis.technicalDetails.numericVariablesUsed;
return {
hasNaturalClusters: true,
optimalClusters: clusteringAnalysis.optimalClusters,
qualityScore: silhouetteScore,
dominantVariables,
clusterProfiles,
};
}
catch (error) {
logger_1.logger.warn('Could not extract clustering insights for visualization:', error);
return { hasNaturalClusters: false };
}
}
/**
* Generate overall visualization strategy based on data characteristics
*/
generateVisualizationStrategy(section1Result, section3Result) {
const columnCount = section1Result.overview.structuralDimensions.totalColumns || 0;
const rowCount = section1Result.overview.structuralDimensions.totalDataRows || 0;
const dataSize = this.determineDataSize(rowCount);
// Determine complexity based on data characteristics
let complexity = types_1.ComplexityLevel.SIMPLE;
if (columnCount > 10 || rowCount > 100000) {
complexity = types_1.ComplexityLevel.MODERATE;
}
if (columnCount > 20 || rowCount > 1000000) {
complexity = types_1.ComplexityLevel.COMPLEX;
}
// Determine interactivity level
let interactivity = types_1.InteractivityLevel.STATIC;
if (dataSize === types_1.DataSize.MEDIUM && columnCount > 5) {
interactivity = types_1.InteractivityLevel.BASIC;
}
if (dataSize === types_1.DataSize.LARGE && columnCount > 10) {
interactivity = types_1.InteractivityLevel.INTERACTIVE;
}
// Determine performance requirements
let performance = types_1.PerformanceLevel.FAST;
if (dataSize === types_1.DataSize.LARGE) {
performance = types_1.PerformanceLevel.MODERATE;
}
if (dataSize === types_1.DataSize.VERY_LARGE) {
performance = types_1.PerformanceLevel.INTENSIVE;
}
const primaryObjectives = this.determinePrimaryObjectives(section3Result);
return {
approach: 'Data-driven chart selection with accessibility and performance optimization',
primaryObjectives,
targetAudience: 'Data analysts, business stakeholders, and decision makers',
complexity,
interactivity,
accessibility: this.config.accessibilityLevel,
performance,
};
}
/**
* Generate univariate recommendations for each column
*/
generateUnivariateRecommendations(_section1Result, section3Result) {
const profiles = [];
if (!section3Result.edaAnalysis?.univariateAnalysis) {
this.warnings.push({
type: 'data_quality',
severity: 'high',
message: 'No univariate analysis data available from Section 3',
recommendation: 'Run Section 3 EDA analysis first',
impact: 'Limited visualization recommendations possible',
});
return profiles;
}
for (const columnAnalysis of section3Result.edaAnalysis.univariateAnalysis) {
const profile = this.createColumnVisualizationProfile(columnAnalysis);
profiles.push(profile);
}
return profiles;
}
/**
* Create visualization profile for a single column
*/
createColumnVisualizationProfile(columnAnalysis) {
const dataType = columnAnalysis.detectedDataType;
const cardinality = columnAnalysis.uniqueValues || 0;
const completeness = 100 - (columnAnalysis.missingPercentage || 0);
// Generate distribution characteristics for numerical columns
let distribution;
if (this.isNumericalType(dataType) && columnAnalysis.distributionAnalysis) {
distribution = {
shape: this.mapSkewnessToShape(columnAnalysis.distributionAnalysis.skewness || 0),
skewness: columnAnalysis.distributionAnalysis.skewness || 0,
kurtosis: columnAnalysis.distributionAnalysis.kurtosis || 0,
outliers: {
count: columnAnalysis.outlierAnalysis?.summary?.totalOutliers || 0,
percentage: columnAnalysis.outlierAnalysis?.summary?.totalPercentage || 0,
extreme: (columnAnalysis.outlierAnalysis?.summary?.totalPercentage || 0) > 10,
impact: this.assessOutlierImpact(columnAnalysis.outlierAnalysis?.summary?.totalPercentage || 0),
},
modality: 'unimodal', // Simplified for now
};
}
// Generate chart recommendations for this column
const recommendations = this.generateColumnChartRecommendations(columnAnalysis, dataType, cardinality, completeness, distribution);
return {
columnName: columnAnalysis.columnName,
dataType,
semanticType: columnAnalysis.inferredSemanticType || 'unknown',
cardinality,
uniqueness: columnAnalysis.uniquePercentage || 0,
completeness,
distribution,
recommendations,
warnings: this.generateColumnWarnings(columnAnalysis, cardinality, completeness),
};
}
/**
* Generate chart recommendations for a specific column
*/
generateColumnChartRecommendations(columnAnalysis, dataType, cardinality, completeness, distribution) {
const recommendations = [];
switch (dataType) {
case types_2.EdaDataType.NUMERICAL_INTEGER:
case types_2.EdaDataType.NUMERICAL_FLOAT:
recommendations.push(...this.generateNumericalRecommendations(columnAnalysis, distribution));
break;
case types_2.EdaDataType.CATEGORICAL:
recommendations.push(...this.generateCategoricalRecommendations(columnAnalysis, cardinality));
break;
case types_2.EdaDataType.DATE_TIME:
recommendations.push(...this.generateDateTimeRecommendations(columnAnalysis));
break;
case types_2.EdaDataType.BOOLEAN:
recommendations.push(...this.generateBooleanRecommendations(columnAnalysis));
break;
case types_2.EdaDataType.TEXT_GENERAL:
case types_2.EdaDataType.TEXT_ADDRESS:
recommendations.push(...this.generateTextRecommendations(columnAnalysis));
break;
default:
this.warnings.push({
type: 'interpretation',
severity: 'medium',
message: `Unknown data type: ${dataType} for column ${columnAnalysis.columnName}`,
recommendation: 'Treating as categorical for visualization purposes',
impact: 'Suboptimal chart recommendations',
});
recommendations.push(...this.generateCategoricalRecommendations(columnAnalysis, cardinality));
}
// Apply quality, accessibility filters, and anti-pattern detection
const filteredRecommendations = recommendations.filter((rec) => this.meetsQualityThreshold(rec, completeness));
const enhancedRecommendations = this.applyAntiPatternDetection(filteredRecommendations, {
cardinality,
completeness,
dataType,
});
return enhancedRecommendations
.slice(0, this.config.maxRecommendationsPerChart)
.sort((a, b) => b.confidence - a.confidence);
}
/**
* Generate recommendations for numerical columns
*/
generateNumericalRecommendations(columnAnalysis, distribution) {
const recommendations = [];
// Histogram - Primary recommendation for distribution
recommendations.push({
chartType: types_1.ChartType.HISTOGRAM,
purpose: types_1.ChartPurpose.DISTRIBUTION,
priority: types_1.RecommendationPriority.PRIMARY,
confidence: 0.9,
reasoning: 'Histograms effectively show the distribution of numerical data, revealing shape, central tendency, and spread',
encoding: this.createNumericalHistogramEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.HISTOGRAM),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.HISTOGRAM),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.HISTOGRAM),
dataPreparation: this.createNumericalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.HISTOGRAM),
});
// Box plot - Good for outlier detection
const outlierPercentage = distribution?.outliers.percentage || 0;
recommendations.push({
chartType: types_1.ChartType.BOX_PLOT,
purpose: types_1.ChartPurpose.OUTLIER_DETECTION,
priority: outlierPercentage > 5 ? types_1.RecommendationPriority.PRIMARY : types_1.RecommendationPriority.SECONDARY,
confidence: outlierPercentage > 5 ? 0.85 : 0.7,
reasoning: outlierPercentage > 5
? 'Box plots excel at highlighting outliers and quartile distribution'
: 'Box plots provide a compact view of distribution and quartiles',
encoding: this.createBoxPlotEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.BOX_PLOT),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.BOX_PLOT),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.BOX_PLOT),
dataPreparation: this.createNumericalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.BOX_PLOT),
});
// Density plot for smooth distribution
if (columnAnalysis.totalValues > 100) {
recommendations.push({
chartType: types_1.ChartType.DENSITY_PLOT,
purpose: types_1.ChartPurpose.DISTRIBUTION,
priority: types_1.RecommendationPriority.ALTERNATIVE,
confidence: 0.75,
reasoning: 'Density plots provide smooth estimates of distribution shape, useful for larger datasets',
encoding: Section4Analyzer.createDensityPlotEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.DENSITY_PLOT),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.DENSITY_PLOT),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.DENSITY_PLOT),
dataPreparation: this.createNumericalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.DENSITY_PLOT),
});
}
// Enhanced Violin Plot with Embedded Box Plot (per specification)
if (columnAnalysis.totalValues > 200) {
const priority = distribution && distribution.outliers.impact === 'high'
? types_1.RecommendationPriority.PRIMARY
: types_1.RecommendationPriority.SECONDARY;
recommendations.push({
chartType: types_1.ChartType.VIOLIN_WITH_BOX,
purpose: types_1.ChartPurpose.DISTRIBUTION,
priority,
confidence: 0.88,
reasoning: 'Violin plot with embedded box plot provides rich distributional comparison showing probability density, median, quartiles, and outliers simultaneously',
encoding: Section4Analyzer.createViolinWithBoxEncoding(columnAnalysis),
interactivity: Section4Analyzer.createAdvancedInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.VIOLIN_WITH_BOX),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.VIOLIN_WITH_BOX),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.VIOLIN_WITH_BOX),
dataPreparation: this.createNumericalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.VIOLIN_WITH_BOX),
});
}
return recommendations;
}
/**
* Generate recommendations for categorical columns
*/
generateCategoricalRecommendations(columnAnalysis, cardinality) {
const recommendations = [];
// Bar chart - Primary for most categorical data
recommendations.push({
chartType: cardinality > 8 ? types_1.ChartType.HORIZONTAL_BAR : types_1.ChartType.BAR_CHART,
purpose: types_1.ChartPurpose.COMPARISON,
priority: types_1.RecommendationPriority.PRIMARY,
confidence: 0.9,
reasoning: cardinality > 8
? 'Horizontal bar charts handle long category labels better and improve readability'
: 'Bar charts excel at comparing categorical data frequencies',
encoding: this.createCategoricalBarEncoding(columnAnalysis, cardinality),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.BAR_CHART),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.BAR_CHART),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.BAR_CHART),
dataPreparation: this.createCategoricalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.BAR_CHART),
});
// Pie chart - Only for low cardinality and composition view
if (cardinality <= 6) {
recommendations.push({
chartType: types_1.ChartType.PIE_CHART,
purpose: types_1.ChartPurpose.COMPOSITION,
priority: types_1.RecommendationPriority.SECONDARY,
confidence: 0.6,
reasoning: 'Pie charts work well for showing parts of a whole when there are few categories',
encoding: Section4Analyzer.createPieChartEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.PIE_CHART),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.PIE_CHART),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.PIE_CHART),
dataPreparation: this.createCategoricalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.PIE_CHART),
});
}
// Treemap for high cardinality
if (cardinality > 20) {
recommendations.push({
chartType: types_1.ChartType.TREEMAP,
purpose: types_1.ChartPurpose.COMPOSITION,
priority: types_1.RecommendationPriority.ALTERNATIVE,
confidence: 0.7,
reasoning: 'Treemaps efficiently display hierarchical data with many categories using space-filling visualization',
encoding: Section4Analyzer.createTreemapEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.TREEMAP),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.TREEMAP),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.TREEMAP),
dataPreparation: this.createCategoricalDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.TREEMAP),
});
}
return recommendations;
}
/**
* Generate recommendations for datetime columns
*/
generateDateTimeRecommendations(columnAnalysis) {
const recommendations = [];
// Time series line chart - Primary for temporal data
recommendations.push({
chartType: types_1.ChartType.TIME_SERIES_LINE,
purpose: types_1.ChartPurpose.TREND,
priority: types_1.RecommendationPriority.PRIMARY,
confidence: 0.9,
reasoning: 'Line charts are optimal for showing temporal trends and patterns over time',
encoding: Section4Analyzer.createTimeSeriesEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.TIME_SERIES_LINE),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.TIME_SERIES_LINE),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.TIME_SERIES_LINE),
dataPreparation: Section4Analyzer.createDateTimeDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.TIME_SERIES_LINE),
});
return recommendations;
}
/**
* Generate recommendations for boolean columns
*/
generateBooleanRecommendations(columnAnalysis) {
const recommendations = [];
// Simple bar chart for boolean distribution
recommendations.push({
chartType: types_1.ChartType.BAR_CHART,
purpose: types_1.ChartPurpose.COMPARISON,
priority: types_1.RecommendationPriority.PRIMARY,
confidence: 0.85,
reasoning: 'Bar charts clearly show the distribution between true/false values',
encoding: Section4Analyzer.createBooleanBarEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.BAR_CHART),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.BAR_CHART),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.BAR_CHART),
dataPreparation: Section4Analyzer.createBooleanDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.BAR_CHART),
});
return recommendations;
}
/**
* Generate recommendations for text columns
*/
generateTextRecommendations(columnAnalysis) {
const recommendations = [];
// Word frequency analysis as horizontal bar chart
if (columnAnalysis.topFrequentWords && columnAnalysis.topFrequentWords.length > 0) {
recommendations.push({
chartType: types_1.ChartType.HORIZONTAL_BAR,
purpose: types_1.ChartPurpose.RANKING,
priority: types_1.RecommendationPriority.PRIMARY,
confidence: 0.8,
reasoning: 'Horizontal bar charts effectively display word frequency rankings from text analysis',
encoding: Section4Analyzer.createTextFrequencyEncoding(columnAnalysis),
interactivity: this.createBasicInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.HORIZONTAL_BAR),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.HORIZONTAL_BAR),
libraryRecommendations: this.getLibraryRecommendations(types_1.ChartType.HORIZONTAL_BAR),
dataPreparation: Section4Analyzer.createTextDataPreparation(columnAnalysis),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.HORIZONTAL_BAR),
});
}
return recommendations;
}
/**
* Generate bivariate recommendations with correlation analysis
*/
generateBivariateRecommendations(section3Result) {
const profiles = [];
if (!section3Result.edaAnalysis?.bivariateAnalysis?.numericalVsNumerical?.correlationPairs) {
logger_1.logger.info('No correlation data available for bivariate analysis');
return profiles;
}
// Extract correlation data from Section 3 results
const correlations = this.extractCorrelations(section3Result);
// Filter for meaningful correlations - exclude ID field and use more practical thresholds
const significantCorrelations = correlations.filter((corr) =>
// Exclude ID field from visualization recommendations
!corr.variable1.toLowerCase().includes('id') &&
!corr.variable2.toLowerCase().includes('id') &&
// Use more practical correlation threshold for medical data
Math.abs(corr.strength) > 0.2 &&
// Be more lenient with p-values
corr.significance <= 0.1);
// Generate recommendations for each significant correlation
for (const correlation of significantCorrelations) {
const profile = Section4Analyzer.createBivariateProfile(correlation, section3Result);
if (profile) {
profiles.push(profile);
}
}
// Sort by correlation strength (descending)
profiles.sort((a, b) => Math.abs(b.strength) - Math.abs(a.strength));
return profiles.slice(0, 10); // Limit to top 10 relationships
}
/**
* Extract correlation data from Section 3 results
*/
extractCorrelations(section3Result) {
const correlations = [];
const correlationPairs = section3Result.edaAnalysis?.bivariateAnalysis?.numericalVsNumerical?.correlationPairs;
if (!correlationPairs) {
return correlations;
}
// Parse correlation pairs to extract pairwise correlations
for (const correlation of correlationPairs) {
const analysis = {
variable1: correlation.variable1,
variable2: correlation.variable2,
correlationType: 'pearson', // Default from Section 3
strength: correlation.correlation,
significance: correlation.pValue || 0.01,
confidenceInterval: [correlation.correlation - 0.1, correlation.correlation + 0.1],
relationshipType: this.determineRelationshipType(correlation.correlation),
visualizationSuitability: this.calculateVisualizationSuitability(correlation),
};
correlations.push(analysis);
}
return correlations;
}
/**
* Create bivariate visualization profile for a correlation
*/
createBivariateProfile(correlation, section3Result) {
// Get data types for both variables
const var1Data = this.getVariableData(correlation.variable1, section3Result);
const var2Data = this.getVariableData(correlation.variable2, section3Result);
if (!var1Data || !var2Data) {
return null;
}
// Generate chart recommendations based on data types and correlation
const recommendations = Section4Analyzer.generateBivariateChartRecommendations(correlation);
return {
variable1: correlation.variable1,
variable2: correlation.variable2,
relationshipType: Section4Analyzer.determineRelationshipType(var1Data.dataType || var1Data.type || 'numerical', var2Data.dataType || var2Data.type || 'numerical', correlation.relationshipType),
strength: correlation.strength,
significance: correlation.significance,
recommendations,
dataPreparation: Section4Analyzer.createBivariateDataPreparation(),
};
}
/**
* Generate chart recommendations for bivariate relationships
*/
generateBivariateChartRecommendations(correlation, var1Data, var2Data) {
const recommendations = [];
const isVar1Numerical = this.isNumericalType(var1Data.dataType);
const isVar2Numerical = this.isNumericalType(var2Data.dataType);
const isVar1Categorical = var1Data.dataType === types_2.EdaDataType.CATEGORICAL;
const isVar2Categorical = var2Data.dataType === types_2.EdaDataType.CATEGORICAL;
// Numerical vs Numerical - Scatter plot with correlation
if (isVar1Numerical && isVar2Numerical) {
recommendations.push({
chartType: types_1.ChartType.SCATTER_PLOT,
purpose: types_1.ChartPurpose.RELATIONSHIP,
priority: types_1.RecommendationPriority.PRIMARY,
confidence: 0.9,
reasoning: `Scatter plot ideal for showing ${correlation.relationshipType.replace('_', ' ')} relationship (r=${correlation.strength.toFixed(3)})`,
encoding: Section4Analyzer.createScatterPlotEncoding(correlation.variable1, correlation.variable2),
interactivity: Section4Analyzer.createAdvancedInteractivity(),
accessibility: Section4Analyzer.createAccessibilityGuidance(types_1.ChartType.SCATTER_PLOT),
performance: Section4Analyzer.createPerformanceConsiderations(types_1.ChartType.SCATTER_PLOT),
libraryRecommendations: Section4Analyzer.getOptimalLibraryRecommendations(types_1.ChartType.SCATTER_PLOT),
dataPreparation: Section4Analyzer.createBivariateDataPreparation(),
designGuidelines: Section4Analyzer.createDesignGuidelines(types_1.ChartType.SCATTER_PLOT),
});
// Add regression line if strong linear relationship
if (Math.abs(correlation.strength) > 0.7 &&
(correlation.relationshipType === types_1.RelationshipType.LINEAR_POSITIVE ||
correlation.relationshipType === types_1.RelationshipType.LINEAR_NEGATIVE)) {
recommendations.push({
chartType: types_1.ChartType.REGRESSION_PLOT,
purpose: types_1.ChartPurpose.RELATIONSHIP,
priority: types_1.RecommendationPriority.SECONDARY,
confidence: 0.85,
reasoning: 'Regr