datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
1,045 lines • 37.7 kB
JavaScript
"use strict";
/**
* DataPilot Configuration System
* Centralized configuration management for all hardcoded values and thresholds
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConfigFactory = exports.ConfigBuilder = exports.ConfigManager = exports.DEFAULT_CONFIG = void 0;
exports.getConfig = getConfig;
exports.loadConfigFromEnvironment = loadConfigFromEnvironment;
exports.getPresetConfig = getPresetConfig;
exports.getUseCaseConfig = getUseCaseConfig;
/**
* Default configuration values
*/
/**
* Enhanced default configuration with comprehensive settings
*/
exports.DEFAULT_CONFIG = {
performance: {
maxRows: 1000000,
maxFieldSize: 1024 * 1024, // 1MB
memoryThresholdBytes: 1024 * 1024 * 1024, // 1GB
chunkSize: 64 * 1024, // 64KB
sampleSize: 1024 * 1024, // 1MB
adaptiveChunkSizing: true,
maxCollectedRowsMultivariate: 2000,
batchSize: 1000,
performanceMonitoringInterval: 10,
memoryCleanupInterval: 20,
emergencyMemoryThresholdMultiplier: 1.5,
},
statistical: {
significanceLevel: 0.05,
alternativeSignificanceLevels: {
normalityTests: 0.05,
correlationTests: 0.01,
hypothesisTests: 0.05,
outlierDetection: 0.01,
},
confidenceLevel: 0.95,
correlationThresholds: {
weak: 0.3,
moderate: 0.5,
strong: 0.7,
veryStrong: 0.9,
},
outlierThresholds: {
zScoreThreshold: 3.0,
modifiedZScoreThreshold: 3.5,
iqrMultiplier: 1.5,
},
normalityThresholds: {
shapiroWilkMinSample: 3,
shapiroWilkMaxSample: 5000,
jarqueBeraThreshold: 0.05,
ksTestThreshold: 0.05,
},
},
quality: {
qualityWeights: {
completeness: 0.2,
uniqueness: 0.15,
validity: 0.2,
consistency: 0.15,
accuracy: 0.15,
timeliness: 0.05,
integrity: 0.05,
reasonableness: 0.03,
precision: 0.01,
representational: 0.01,
},
qualityThresholds: {
excellent: 90,
good: 80,
fair: 65,
needsImprovement: 45,
},
duplicateThresholds: {
exactDuplicateThreshold: 1.0,
semanticSimilarityThreshold: 0.8,
fuzzyMatchThreshold: 0.7,
},
patternValidation: {
maxViolationsPerPattern: 100,
maxViolationsToTrack: 1000,
enableBuiltInPatterns: true,
},
businessRules: {
maxViolationsToTrack: 1000,
enableBuiltInRules: true,
},
externalValidation: {
maxSampleSize: 1000,
maxExampleViolations: 10,
},
},
analysis: {
maxCategoricalLevels: 50,
maxCorrelationPairs: 50,
samplingThreshold: 10000,
outlierMethods: ['iqr', 'zscore', 'modified_zscore'],
normalityTests: ['shapiro', 'jarque_bera', 'ks_test'],
enableMultivariate: true,
enabledAnalyses: ['univariate', 'bivariate', 'correlations'],
highCardinalityThreshold: 80,
missingValueQualityThreshold: 20,
multivariateThreshold: 1000,
maxDimensionsForPCA: 10,
clusteringMethods: ['kmeans', 'hierarchical'],
},
streaming: {
memoryThresholdMB: 100,
maxRowsAnalyzed: 500000,
adaptiveChunkSizing: {
enabled: true,
minChunkSize: 50,
maxChunkSize: 2000,
reductionFactor: 0.6,
expansionFactor: 1.1,
targetMemoryUtilization: 0.8,
},
memoryManagement: {
cleanupInterval: 20,
emergencyThresholdMultiplier: 1.5,
forceGarbageCollection: true,
gcFrequency: 1000,
memoryLeakDetection: false,
autoGarbageCollect: false,
},
},
visualization: {
maxDataPoints: 10000,
chartScoringWeights: {
dataFit: 0.4,
clarity: 0.3,
insightPotential: 0.2,
accessibility: 0.1,
},
colorPalettes: {
preferColorblindSafe: true,
maxColors: 12,
},
},
modeling: {
maxFeaturesAutoSelection: 100,
algorithmScoringWeights: {
performance: 0.4,
interpretability: 0.3,
scalability: 0.2,
robustness: 0.1,
},
crossValidation: {
defaultFolds: 5,
minSampleSize: 30,
},
featureSelection: {
correlationThreshold: 0.95,
importanceThreshold: 0.01,
},
},
output: {
includeVisualizationRecommendations: true,
includeEngineeringInsights: true,
verboseOutput: false,
progressReporting: true,
format: 'json',
},
// Environment configuration
environment: {
mode: 'development',
performance: {},
statistical: {},
quality: {},
analysis: {},
streaming: {},
visualization: {},
modeling: {},
output: {},
},
// Feature flags
features: {
enableAdvancedMultivariate: true,
enableMLReadinessScoring: true,
enableRealTimeProcessing: false,
enableCloudIntegration: false,
},
// Extensions
extensions: {
customAnalyzers: [],
pluginPaths: [],
externalValidators: {},
},
// Security settings
security: {
enableDataEncryption: false,
redactSensitiveData: true,
auditLogging: false,
maxFileSize: 10 * 1024 * 1024 * 1024, // 10GB
},
};
// Re-export refactored managers
// Configuration managers temporarily disabled pending refactor
/**
* Enhanced Configuration Manager - Refactored as Facade
* Delegates to focused managers for single responsibility
*/
// Config managers with full method implementations
class CoreConfigManager {
config;
constructor(config) {
this.config = { ...exports.DEFAULT_CONFIG, ...config };
}
getConfig() { return this.config; }
getCoreConfig() { return this.config; }
updateConfig(updates) {
if (!updates)
return;
this.config = { ...this.config, ...updates };
}
reset() {
this.config = { ...exports.DEFAULT_CONFIG };
}
getPerformanceConfig() { return JSON.parse(JSON.stringify(this.config.performance)); }
getStatisticalConfig() { return JSON.parse(JSON.stringify(this.config.statistical)); }
getQualityConfig() { return JSON.parse(JSON.stringify(this.config.quality)); }
getAnalysisConfig() { return JSON.parse(JSON.stringify(this.config.analysis)); }
getStreamingConfig() { return JSON.parse(JSON.stringify(this.config.streaming)); }
getVisualizationConfig() { return JSON.parse(JSON.stringify(this.config.visualization)); }
getModelingConfig() { return JSON.parse(JSON.stringify(this.config.modeling)); }
getOutputConfig() { return JSON.parse(JSON.stringify(this.config.output)); }
updatePerformanceConfig(updates) {
if (!updates)
return;
this.config.performance = { ...this.config.performance, ...updates };
}
updateStatisticalConfig(updates) {
if (!updates)
return;
this.config.statistical = { ...this.config.statistical, ...updates };
}
updateQualityConfig(updates) {
if (!updates)
return;
this.config.quality = { ...this.config.quality, ...updates };
}
updateAnalysisConfig(updates) {
if (!updates)
return;
this.config.analysis = { ...this.config.analysis, ...updates };
}
updateStreamingConfig(updates) {
if (!updates)
return;
this.config.streaming = {
...this.config.streaming,
...updates,
// Handle nested updates for adaptiveChunkSizing
adaptiveChunkSizing: updates.adaptiveChunkSizing
? { ...this.config.streaming.adaptiveChunkSizing, ...updates.adaptiveChunkSizing }
: this.config.streaming.adaptiveChunkSizing
};
}
updateVisualizationConfig(updates) {
if (!updates)
return;
this.config.visualization = { ...this.config.visualization, ...updates };
}
updateModelingConfig(updates) {
if (!updates)
return;
this.config.modeling = { ...this.config.modeling, ...updates };
}
updateOutputConfig(updates) {
if (!updates)
return;
this.config.output = { ...this.config.output, ...updates };
}
}
class EnvironmentConfigManager {
environment = 'development';
loadFromEnvironmentVariables() {
const config = {};
// Helper function to safely parse numbers
const parseNumber = (value) => {
if (!value)
return undefined;
const num = Number(value);
return isNaN(num) ? undefined : num;
};
// Helper function to safely parse booleans
const parseBoolean = (value) => {
if (!value)
return undefined;
const lower = value.toLowerCase();
if (lower === 'true')
return true;
if (lower === 'false')
return false;
return undefined;
};
// Helper function to convert MB to bytes
const mbToBytes = (mb) => mb * 1024 * 1024;
// Parse performance settings
const maxRows = parseNumber(process.env.DATAPILOT_MAX_ROWS);
const memoryThresholdMB = parseNumber(process.env.DATAPILOT_MEMORY_THRESHOLD_MB);
if (maxRows !== undefined || memoryThresholdMB !== undefined) {
config.performance = {};
if (maxRows !== undefined) {
config.performance.maxRows = maxRows;
}
if (memoryThresholdMB !== undefined) {
config.performance.memoryThresholdBytes = mbToBytes(memoryThresholdMB);
}
}
// Parse streaming settings
if (memoryThresholdMB !== undefined) {
config.streaming = {
memoryThresholdMB: memoryThresholdMB
};
}
// Parse statistical settings
const significanceLevel = parseNumber(process.env.DATAPILOT_SIGNIFICANCE_LEVEL);
if (significanceLevel !== undefined && significanceLevel >= 0 && significanceLevel <= 1) {
config.statistical = {
significanceLevel: significanceLevel
};
}
// Parse analysis settings
const enableMultivariate = parseBoolean(process.env.DATAPILOT_ENABLE_MULTIVARIATE);
if (enableMultivariate !== undefined) {
config.analysis = {
enableMultivariate: enableMultivariate
};
}
// Parse security settings
const maxFileSizeMB = parseNumber(process.env.DATAPILOT_MAX_FILE_SIZE_MB);
if (maxFileSizeMB !== undefined) {
config.security = {
maxFileSize: mbToBytes(maxFileSizeMB)
};
}
// Handle preset configuration
const preset = process.env.DATAPILOT_PRESET;
if (preset && ['small', 'medium', 'large', 'xlarge'].includes(preset)) {
const runtimeManager = new RuntimeConfigManager();
const presetConfig = runtimeManager.getPresetConfig(preset);
// Merge preset config with environment overrides, giving priority to environment
return this.deepMergeConfigs(presetConfig, config);
}
return config;
}
deepMergeConfigs(target, source) {
const result = JSON.parse(JSON.stringify(target));
for (const key in source) {
if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
result[key] = this.deepMergeConfigs(result[key] || {}, source[key]);
}
else {
result[key] = source[key];
}
}
return result;
}
setEnvironment(env) {
this.environment = env;
}
getEnvironmentConfig(env) {
const environmentConfigs = {
'development': {
performance: {
maxRows: 50000
},
streaming: {
memoryThresholdMB: 50,
adaptiveChunkSizing: {
enabled: false
}
}
},
'production': {
performance: {
maxRows: 2000000
},
streaming: {
memoryThresholdMB: 500,
adaptiveChunkSizing: {
enabled: true
}
}
},
'ci': {
performance: {
maxRows: 10000
},
analysis: {
enableMultivariate: false,
enabledAnalyses: ['univariate']
}
},
'test': {
performance: {
maxRows: 1000
},
statistical: {
significanceLevel: 0.1,
confidenceLevel: 0.9
}
}
};
return this.deepMerge({}, environmentConfigs[env] || {});
}
deepMerge(target, source) {
const result = JSON.parse(JSON.stringify(target));
for (const key in source) {
if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
result[key] = this.deepMerge(result[key] || {}, source[key]);
}
else {
result[key] = source[key];
}
}
return result;
}
getPerformancePresetConfig(preset) {
const presetConfigs = {
'low-memory': {
performance: {
maxRows: 10000,
memoryThresholdBytes: 256 * 1024 * 1024,
adaptiveChunkSizing: false
},
streaming: {
memoryThresholdMB: 64
}
},
'balanced': {
performance: {
maxRows: 100000,
memoryThresholdBytes: 1024 * 1024 * 1024,
adaptiveChunkSizing: true
}
},
'high-performance': {
performance: {
maxRows: 1000000,
memoryThresholdBytes: 4096 * 1024 * 1024,
batchSize: 5000
}
},
'custom': {
performance: {
maxRows: 500000
}
}
};
return this.deepMerge({}, presetConfigs[preset] || {});
}
validateConfig(config) {
const errors = [];
const warnings = [];
// Performance validation
if (config.performance.maxRows <= 0) {
errors.push('performance.maxRows: maxRows must be a positive number');
}
if (config.performance.chunkSize < 1024) {
errors.push('performance.chunkSize: chunkSize must be at least 1024 bytes');
}
if (config.performance.memoryThresholdBytes < 256 * 1024 * 1024) {
warnings.push('Memory threshold is quite low, consider increasing for better performance');
}
if (config.performance.chunkSize < 32 * 1024) {
warnings.push('Small chunk size may impact performance, consider increasing');
}
if (config.performance.batchSize > config.performance.chunkSize / 64) {
warnings.push('Batch size seems large relative to chunk size');
}
// Statistical validation
if (config.statistical.significanceLevel <= 0 || config.statistical.significanceLevel >= 1) {
errors.push('statistical.significanceLevel: significanceLevel must be between 0 and 1');
}
if (config.statistical.confidenceLevel <= 0 || config.statistical.confidenceLevel >= 1) {
errors.push('statistical.confidenceLevel: confidenceLevel must be between 0 and 1');
}
// Correlation test validation - check if correlation test significance is more strict than base
if (config.statistical.alternativeSignificanceLevels.correlationTests < config.statistical.significanceLevel) {
warnings.push('correlationTests significance level is more strict than base level');
}
// Quality validation with null/undefined safety
const qualityWeights = config.quality?.qualityWeights;
if (qualityWeights && typeof qualityWeights === 'object') {
const weights = Object.values(qualityWeights);
if (weights.length > 0) {
const weightSum = weights.reduce((a, b) => (typeof a === 'number' ? a : 0) + (typeof b === 'number' ? b : 0), 0);
if (Math.abs(weightSum - 1.0) > 0.1) {
// Significantly off from 1.0 - return errors, not warnings
errors.push('quality.qualityWeights: qualityWeights must sum to 1.0');
}
else if (Math.abs(weightSum - 1.0) > 0.01) {
warnings.push(`quality.qualityWeights sum to ${weightSum.toFixed(3)}, consider adjusting to sum to 1.0 for optimal scoring`);
}
}
}
// Cross-section validation
if (config.streaming.memoryThresholdMB * 1024 * 1024 > config.performance.memoryThresholdBytes) {
warnings.push('streaming.memoryThresholdMB should not exceed performance memory threshold');
}
if (config.streaming.maxRowsAnalyzed > config.performance.maxRows) {
warnings.push('streaming.maxRowsAnalyzed should not exceed performance.maxRows');
}
return {
isValid: errors.length === 0,
errors,
warnings
};
}
}
class RuntimeConfigManager {
runtimeOverrides = {};
adaptiveCache = new Map();
deepMerge(target, source) {
const result = JSON.parse(JSON.stringify(target));
for (const key in source) {
if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
result[key] = this.deepMerge(result[key] || {}, source[key]);
}
else {
result[key] = source[key];
}
}
return result;
}
getPresetConfig(preset) {
const presetConfigs = {
'small': {
performance: {
maxRows: 10000,
},
streaming: {
memoryThresholdMB: 25,
},
analysis: {
enableMultivariate: false,
},
},
'medium': {
performance: {
maxRows: 100000,
},
streaming: {
memoryThresholdMB: 50,
},
analysis: {
enableMultivariate: true,
},
},
'large': {
performance: {
maxRows: 1000000,
},
streaming: {
memoryThresholdMB: 100,
adaptiveChunkSizing: {
expansionFactor: 1.2,
},
},
},
'xlarge': {
performance: {
maxRows: 5000000,
},
streaming: {
memoryThresholdMB: 200,
memoryManagement: {
cleanupInterval: 10,
},
},
analysis: {
samplingThreshold: 50000,
},
},
};
const presetConfig = presetConfigs[preset];
return presetConfig ? this.deepMerge(exports.DEFAULT_CONFIG, presetConfig) : JSON.parse(JSON.stringify(exports.DEFAULT_CONFIG));
}
getUseCaseConfig(useCase) {
const useCaseConfigs = {
'data-quality': {
quality: {
qualityWeights: {
completeness: 0.25,
uniqueness: 0.15,
validity: 0.2,
consistency: 0.15,
accuracy: 0.15,
timeliness: 0.05,
integrity: 0.05,
reasonableness: 0.03,
precision: 0.01,
representational: 0.01,
},
},
analysis: {
enabledAnalyses: ['univariate'],
},
},
'eda-focused': {
analysis: {
enableMultivariate: true,
maxCorrelationPairs: 100,
},
statistical: {
significanceLevel: 0.01,
},
},
'ml-pipeline': {
analysis: {
enableMultivariate: true,
},
modeling: {
algorithmScoringWeights: {
performance: 0.5,
interpretability: 0.3,
scalability: 0.2,
robustness: 0.0,
},
},
},
'visualization': {
visualization: {
maxDataPoints: 50000,
chartScoringWeights: {
dataFit: 0.3,
clarity: 0.3,
insightPotential: 0.2,
accessibility: 0.2,
},
},
},
'quick-scan': {
performance: {
maxRows: 5000,
},
analysis: {
enableMultivariate: false,
maxCorrelationPairs: 10,
},
},
};
const useCaseConfig = useCaseConfigs[useCase];
return useCaseConfig ? this.deepMerge(exports.DEFAULT_CONFIG, useCaseConfig) : JSON.parse(JSON.stringify(exports.DEFAULT_CONFIG));
}
mergeConfigs(base, override) {
return { ...base, ...override };
}
applyRuntimeOverrides(config) {
return { ...config, ...this.runtimeOverrides };
}
clearRuntimeOverrides() {
this.runtimeOverrides = {};
}
clearAdaptiveCache() {
this.adaptiveCache.clear();
}
getAdaptiveThresholds(datasetSize, memoryAvailable, config) {
const cacheKey = `${datasetSize}-${memoryAvailable}`;
if (this.adaptiveCache.has(cacheKey)) {
return this.adaptiveCache.get(cacheKey);
}
const result = {};
// Convert memory to MB for easier calculation
const memoryMB = memoryAvailable / (1024 * 1024);
// Define memory thresholds (in MB)
const LOW_MEMORY_THRESHOLD = 512; // 512MB
const HIGH_MEMORY_THRESHOLD = 2048; // 2GB
// Define dataset size categories
const SMALL_DATASET = 10000;
const MEDIUM_DATASET = 100000;
const LARGE_DATASET = 1000000;
const VERY_LARGE_DATASET = 10000000;
// Determine memory category
const isLowMemory = memoryMB < LOW_MEMORY_THRESHOLD;
const isHighMemory = memoryMB >= HIGH_MEMORY_THRESHOLD;
// Determine dataset category
const isSmallDataset = datasetSize <= SMALL_DATASET;
const isMediumDataset = datasetSize > SMALL_DATASET && datasetSize <= MEDIUM_DATASET;
const isLargeDataset = datasetSize > MEDIUM_DATASET && datasetSize <= LARGE_DATASET;
const isVeryLargeDataset = datasetSize > LARGE_DATASET;
// Performance configuration based on dataset size and memory
if (isVeryLargeDataset && isHighMemory) {
// Very large dataset with high memory (like 100M rows, 8GB)
result.performance = {
maxRows: Math.min(2000000, datasetSize),
chunkSize: 128 * 1024, // 128KB
batchSize: 2000
};
}
else if (isLargeDataset && isHighMemory) {
// Large dataset with high memory (like 2M rows, 2GB)
result.performance = {
maxRows: Math.min(2000000, datasetSize),
chunkSize: 128 * 1024, // 128KB
batchSize: 2000
};
}
else if (isSmallDataset) {
// Small dataset (like 5K rows)
result.performance = {
chunkSize: 16 * 1024, // 16KB
batchSize: 100
};
}
else if (isLowMemory) {
// Low memory scenarios (like 100K rows, 256MB)
result.streaming = {
memoryThresholdMB: 50,
maxRowsAnalyzed: 100000,
adaptiveChunkSizing: {
enabled: true,
minChunkSize: 50,
maxChunkSize: 500,
reductionFactor: 0.6,
expansionFactor: 1.1,
targetMemoryUtilization: 0.8
}
};
result.performance = {
maxCollectedRowsMultivariate: 500
};
}
// Note: For medium datasets with adequate memory, no adaptive configuration is needed
// Additional adaptive settings for extreme cases
if (isVeryLargeDataset && isLowMemory) {
// Very large dataset with low memory - aggressive constraints
result.performance = {
maxRows: 50000,
chunkSize: 8 * 1024, // 8KB
batchSize: 50
};
result.streaming = {
memoryThresholdMB: 25,
maxRowsAnalyzed: 50000,
adaptiveChunkSizing: {
enabled: true,
minChunkSize: 50,
maxChunkSize: 200,
reductionFactor: 0.5,
expansionFactor: 1.1,
targetMemoryUtilization: 0.8
}
};
}
// Cache the result
this.adaptiveCache.set(cacheKey, result);
return result;
}
}
class ConfigManager {
static instance;
coreConfigManager; // Will be CoreConfigManager
environmentConfigManager; // Will be EnvironmentConfigManager
runtimeConfigManager; // Will be RuntimeConfigManager
constructor(initialConfig) {
// Initialize stub managers
this.coreConfigManager = new CoreConfigManager(initialConfig);
this.environmentConfigManager = new EnvironmentConfigManager();
this.runtimeConfigManager = new RuntimeConfigManager();
// Apply environment config if specified
if (initialConfig?.environment?.mode) {
this.applyEnvironmentConfig(initialConfig.environment.mode);
}
}
static getInstance(initialConfig) {
if (!ConfigManager.instance) {
ConfigManager.instance = new ConfigManager(initialConfig);
}
return ConfigManager.instance;
}
/**
* Apply environment-specific configuration
*/
applyEnvironmentConfig(environment) {
this.environmentConfigManager.setEnvironment(environment);
const envConfig = this.environmentConfigManager.getEnvironmentConfig(environment);
if (envConfig) {
this.updateConfig(envConfig);
}
}
/**
* Apply performance preset with type safety
*/
applyPerformancePreset(preset) {
if (preset.preset === 'custom') {
// For custom presets, use the provided config directly through the performance updater
if (preset.config) {
this.updatePerformanceConfig(preset.config);
}
}
else {
// For predefined presets, get the configuration by preset name
const presetConfig = this.environmentConfigManager.getPerformancePresetConfig(preset.preset);
this.updateConfig(presetConfig);
}
}
/**
* Get the complete configuration
*/
getConfig() {
// Get core config
const coreConfig = this.coreConfigManager.getCoreConfig();
// Apply runtime overrides
const mergedConfig = this.runtimeConfigManager.mergeConfigs(exports.DEFAULT_CONFIG, coreConfig);
// Apply any runtime overrides
return this.runtimeConfigManager.applyRuntimeOverrides(mergedConfig);
}
/**
* Get a specific configuration section - Delegate to CoreConfigManager
*/
getPerformanceConfig() {
return this.coreConfigManager.getPerformanceConfig();
}
getStatisticalConfig() {
return this.coreConfigManager.getStatisticalConfig();
}
getQualityConfig() {
return this.coreConfigManager.getQualityConfig();
}
getAnalysisConfig() {
return this.coreConfigManager.getAnalysisConfig();
}
getStreamingConfig() {
return this.coreConfigManager.getStreamingConfig();
}
getVisualizationConfig() {
return this.coreConfigManager.getVisualizationConfig();
}
getModelingConfig() {
return this.coreConfigManager.getModelingConfig();
}
getOutputConfig() {
return this.coreConfigManager.getOutputConfig();
}
/**
* Update configuration dynamically
*/
updateConfig(updates) {
if (!updates)
return;
// Update core config sections
if (updates.performance) {
this.coreConfigManager.updatePerformanceConfig(updates.performance);
}
if (updates.statistical) {
this.coreConfigManager.updateStatisticalConfig(updates.statistical);
}
if (updates.quality) {
this.coreConfigManager.updateQualityConfig(updates.quality);
}
if (updates.analysis) {
this.coreConfigManager.updateAnalysisConfig(updates.analysis);
}
if (updates.streaming) {
this.coreConfigManager.updateStreamingConfig(updates.streaming);
}
if (updates.visualization) {
this.coreConfigManager.updateVisualizationConfig(updates.visualization);
}
if (updates.modeling) {
this.coreConfigManager.updateModelingConfig(updates.modeling);
}
if (updates.output) {
this.coreConfigManager.updateOutputConfig(updates.output);
}
// Handle environment updates
if (updates.environment?.mode) {
this.applyEnvironmentConfig(updates.environment.mode);
}
}
/**
* Update specific configuration sections - Delegate to CoreConfigManager
*/
updatePerformanceConfig(updates) {
this.coreConfigManager.updatePerformanceConfig(updates);
}
updateStatisticalConfig(updates) {
this.coreConfigManager.updateStatisticalConfig(updates);
}
updateQualityConfig(updates) {
this.coreConfigManager.updateQualityConfig(updates);
}
updateAnalysisConfig(updates) {
this.coreConfigManager.updateAnalysisConfig(updates);
}
updateStreamingConfig(updates) {
this.coreConfigManager.updateStreamingConfig(updates);
}
updateVisualizationConfig(updates) {
this.coreConfigManager.updateVisualizationConfig(updates);
}
updateModelingConfig(updates) {
this.coreConfigManager.updateModelingConfig(updates);
}
updateOutputConfig(updates) {
this.coreConfigManager.updateOutputConfig(updates);
}
/**
* Reset to default configuration
*/
reset() {
this.coreConfigManager.reset();
this.runtimeConfigManager.clearRuntimeOverrides();
this.runtimeConfigManager.clearAdaptiveCache();
}
/**
* Get adaptive thresholds based on dataset characteristics - Delegate to RuntimeConfigManager
*/
getAdaptiveThresholds(datasetSize, memoryAvailable) {
return this.runtimeConfigManager.getAdaptiveThresholds(datasetSize, memoryAvailable, this.getConfig());
}
/**
* Comprehensive configuration validation - Delegate to EnvironmentConfigManager
*/
validateConfig() {
return this.environmentConfigManager.validateConfig(this.getConfig());
}
}
exports.ConfigManager = ConfigManager;
/**
* Convenience function to get the global config manager
*/
function getConfig() {
return ConfigManager.getInstance();
}
/**
* Create a configuration builder for fluent API
*/
class ConfigBuilder {
config = {};
static create() {
return new ConfigBuilder();
}
deepMergeConfigs(target, source) {
const result = JSON.parse(JSON.stringify(target));
for (const key in source) {
if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
result[key] = this.deepMergeConfigs(result[key] || {}, source[key]);
}
else {
result[key] = source[key];
}
}
return result;
}
environment(mode) {
const manager = ConfigManager.getInstance();
manager.applyEnvironmentConfig(mode);
return this;
}
preset(presetName) {
// Config manager temporarily disabled
const runtimeManager = new RuntimeConfigManager();
const presetConfig = runtimeManager.getPresetConfig(presetName);
// Ensure we merge with DEFAULT_CONFIG as base to avoid missing properties
this.config = this.deepMergeConfigs(this.config, presetConfig);
return this;
}
useCase(useCase) {
// Config manager temporarily disabled
const runtimeManager = new RuntimeConfigManager();
const useCaseConfig = runtimeManager.getUseCaseConfig(useCase);
// Ensure we merge with DEFAULT_CONFIG as base to avoid missing properties
this.config = this.deepMergeConfigs(this.config, useCaseConfig);
return this;
}
performance(config) {
this.config.performance = this.deepMergeConfigs(this.config.performance || {}, config);
return this;
}
statistical(config) {
this.config.statistical = this.deepMergeConfigs(this.config.statistical || {}, config);
return this;
}
quality(config) {
this.config.quality = this.deepMergeConfigs(this.config.quality || {}, config);
return this;
}
build() {
// Ensure we have a complete configuration by merging with DEFAULT_CONFIG
const completeConfig = this.deepMergeConfigs(exports.DEFAULT_CONFIG, this.config);
// Validate the configuration directly to avoid singleton caching issues
const environmentManager = new EnvironmentConfigManager();
const validation = environmentManager.validateConfig(completeConfig);
if (!validation.isValid) {
throw new Error(`Configuration validation failed: ${validation.errors.join(', ')}`);
}
if (validation.warnings.length > 0) {
console.warn('Configuration warnings:', validation.warnings);
}
// Return the complete configuration we built
return completeConfig;
}
}
exports.ConfigBuilder = ConfigBuilder;
/**
* Type-safe configuration factory functions
*/
exports.ConfigFactory = {
development: () => ConfigBuilder.create().environment('development'),
production: () => ConfigBuilder.create().environment('production'),
ci: () => ConfigBuilder.create().environment('ci'),
test: () => ConfigBuilder.create().environment('test'),
small: () => ConfigBuilder.create().preset('small'),
medium: () => ConfigBuilder.create().preset('medium'),
large: () => ConfigBuilder.create().preset('large'),
xlarge: () => ConfigBuilder.create().preset('xlarge'),
dataQuality: () => ConfigBuilder.create().useCase('data-quality'),
eda: () => ConfigBuilder.create().useCase('eda-focused'),
ml: () => ConfigBuilder.create().useCase('ml-pipeline'),
visualization: () => ConfigBuilder.create().useCase('visualization'),
quickScan: () => ConfigBuilder.create().useCase('quick-scan'),
};
/**
* Enhanced environment-based configuration loading with validation
*/
function loadConfigFromEnvironment() {
// Config manager temporarily disabled
const envManager = new EnvironmentConfigManager();
return envManager.loadFromEnvironmentVariables();
}
/**
* Enhanced dataset size-based configuration presets
*/
function getPresetConfig(presetName) {
// Config manager temporarily disabled
const runtimeManager = new RuntimeConfigManager();
return runtimeManager.getPresetConfig(presetName);
}
/**
* Create configuration for specific use cases
*/
function getUseCaseConfig(useCase) {
// Config manager temporarily disabled
const runtimeManager = new RuntimeConfigManager();
return runtimeManager.getUseCaseConfig(useCase);
}
//# sourceMappingURL=config.js.map