datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
986 lines ā¢ 145 kB
JavaScript
#!/usr/bin/env node
"use strict";
/**
* DataPilot CLI - Main entry point
* A lightweight CLI statistical computation engine for comprehensive CSV data analysis
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.OutputManager = exports.ProgressReporter = exports.ArgumentParser = exports.DataPilotCLI = void 0;
const path_1 = require("path");
const fs_1 = require("fs");
const argument_parser_1 = require("./argument-parser");
Object.defineProperty(exports, "ArgumentParser", { enumerable: true, get: function () { return argument_parser_1.ArgumentParser; } });
const progress_reporter_1 = require("./progress-reporter");
Object.defineProperty(exports, "ProgressReporter", { enumerable: true, get: function () { return progress_reporter_1.ProgressReporter; } });
const output_manager_1 = require("./output-manager");
Object.defineProperty(exports, "OutputManager", { enumerable: true, get: function () { return output_manager_1.OutputManager; } });
const dependency_resolver_1 = require("./dependency-resolver");
const performance_manager_1 = require("./performance-manager");
const overview_1 = require("../analyzers/overview");
const quality_1 = require("../analyzers/quality");
const section2_formatter_1 = require("../analyzers/quality/section2-formatter");
const streaming_analyzer_1 = require("../analyzers/streaming/streaming-analyzer");
const section3_formatter_1 = require("../analyzers/eda/section3-formatter");
const visualization_1 = require("../analyzers/visualization");
const types_1 = require("../analyzers/visualization/types");
const engineering_1 = require("../analyzers/engineering");
const modeling_1 = require("../analyzers/modeling");
const joins_1 = require("../analyzers/joins");
const csv_parser_1 = require("../parsers/csv-parser");
const universal_analyzer_1 = require("./universal-analyzer");
const logger_1 = require("../utils/logger");
const types_2 = require("../core/types");
const error_handler_1 = require("../utils/error-handler");
const validation_1 = require("../utils/validation");
const memory_manager_1 = require("../utils/memory-manager");
const retry_1 = require("../utils/retry");
const types_3 = require("./types");
const types_4 = require("../core/types");
class DataPilotCLI {
argumentParser;
progressReporter;
outputManager;
dependencyResolver;
performanceManager;
dependencyCache = new Map();
errorContext = {};
sectionErrors = new Map();
isInitialized = false;
constructor() {
this.argumentParser = new argument_parser_1.ArgumentParser();
this.progressReporter = new progress_reporter_1.ProgressReporter();
// Initialize error handling and memory management
this.initializeErrorHandling();
}
/**
* Set progress callback for test purposes
*/
setProgressCallback(callback) {
this.progressReporter.setProgressCallback?.(callback);
}
/**
* Register section resolvers for dependency injection
*/
registerSectionResolvers(filePath, options) {
if (!this.dependencyResolver) {
this.dependencyResolver = new dependency_resolver_1.AnalyzerDependencyResolver(this.errorContext);
}
// Section resolvers will be registered as needed by individual analysis methods
}
/**
* Comprehensive input validation with actionable feedback
*/
async validateInputs(filePath, options) {
this.errorContext = {
operation: 'validation',
filePath,
};
// Get command context to determine validation type
const commandContext = this.argumentParser.getLastContext();
const command = commandContext?.command;
// Commands that expect directories instead of files
const directoryCommands = ['discover'];
// Commands that don't require any file input
const noFileCommands = ['perf', 'clear-cache'];
if (noFileCommands.includes(command)) {
// Skip file validation for commands that don't need files
return '';
}
if (directoryCommands.includes(command)) {
// Validate directory instead of file
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
const path = await Promise.resolve().then(() => __importStar(require('path')));
if (!fs.existsSync(filePath)) {
throw new types_3.ValidationError(`Directory not found: ${filePath}`);
}
const stats = fs.statSync(filePath);
if (!stats.isDirectory()) {
throw new types_3.ValidationError(`Path is not a directory: ${filePath}`);
}
return filePath;
}
// Use basic file validation for file-based commands
const basicValidation = this.argumentParser.validateFile(filePath);
// Enhanced validation using new validation system
const enhancedValidation = validation_1.Validator.validateAll(basicValidation, options);
// Report warnings
if (enhancedValidation.warnings.length > 0) {
logger_1.logger.warn(`Found ${enhancedValidation.warnings.length} validation warnings:`, this.errorContext);
enhancedValidation.warnings.forEach((warning) => {
console.warn(`ā ļø ${warning.getFormattedMessage()}`);
const suggestions = warning.getSuggestions();
if (suggestions.length > 0) {
console.warn(' Suggestions:');
suggestions.forEach((suggestion) => console.warn(` ${suggestion}`));
}
});
}
// Handle errors
if (enhancedValidation.errors.length > 0) {
const firstError = enhancedValidation.errors[0];
// Show all errors in verbose mode
if (options.verbose && enhancedValidation.errors.length > 1) {
console.error(`\nā Found ${enhancedValidation.errors.length} validation errors:`);
enhancedValidation.errors.forEach((error, index) => {
console.error(` ${index + 1}. ${error.getFormattedMessage()}`);
});
console.error('\nAddressing the first error:');
}
throw firstError;
}
return basicValidation;
}
/**
* Initialize comprehensive error handling systems
*/
initializeErrorHandling() {
// Start memory monitoring
memory_manager_1.globalMemoryManager.startMonitoring({ analyzer: 'CLI', operation: 'initialization' });
// Register cleanup callbacks
memory_manager_1.globalMemoryManager.registerCleanupCallback(() => {
logger_1.logger.debug('Memory cleanup: clearing dependency cache');
this.dependencyCache.clear();
});
memory_manager_1.globalResourceManager.register('cli-progress', () => {
this.progressReporter.cleanup();
}, 'ui');
memory_manager_1.globalResourceManager.register('cli-output', () => {
if (this.outputManager) {
// Cleanup any open file handles in output manager
logger_1.logger.debug('Cleaning up output manager resources');
}
}, 'io');
// Register global cleanup handler
memory_manager_1.globalCleanupHandler.register(async () => {
logger_1.logger.debug('CLI cleanup: stopping memory monitoring');
memory_manager_1.globalMemoryManager.stopMonitoring();
this.dependencyCache.clear();
});
}
/**
* Main CLI execution entry point
*/
async run(argv = process.argv) {
try {
// Parse command line arguments
const context = this.argumentParser.parse(argv);
// Handle help and version commands
if (context.command === 'help' || context.args.length === 0) {
this.argumentParser.showHelp();
return { success: true, exitCode: 0 };
}
// Get the actual command context from the parser
const commandContext = this.argumentParser.getLastContext();
if (!commandContext) {
// Debug logging for troubleshooting
console.log('Debug: No command context found');
console.log('Debug: Context:', context);
throw new types_3.ValidationError('No command specified');
}
// Configure logger based on CLI options
if (commandContext.options.quiet) {
logger_1.logger.setLevel(logger_1.LogLevel.ERROR);
}
else if (commandContext.options.verbose) {
logger_1.logger.setLevel(logger_1.LogLevel.DEBUG);
}
else {
logger_1.logger.setLevel(logger_1.LogLevel.INFO);
}
// Set up progress reporting and output management with merged options
this.progressReporter = new progress_reporter_1.ProgressReporter(commandContext.options.quiet || false, commandContext.options.verbose || false);
this.outputManager = new output_manager_1.OutputManager(commandContext.options);
// Comprehensive input validation (skip for commands that don't need files)
const noFileCommands = ['perf', 'clear-cache'];
let validatedFilePath = '';
if (!noFileCommands.includes(commandContext.command)) {
validatedFilePath = await this.validateInputs(commandContext.file, commandContext.options);
}
// Execute the appropriate command
const result = await this.executeCommand(commandContext.command, validatedFilePath, commandContext.options, context.startTime, commandContext.args);
// Complete performance monitoring and add metrics to result
return this.completePerformanceMonitoring(result);
}
catch (error) {
return this.handleError(error);
}
finally {
this.progressReporter.cleanup();
}
}
/**
* Generic analysis execution method with comprehensive error handling
*/
async executeGenericAnalysis(config, filePath, options, startTime) {
if (!this.outputManager) {
throw new Error('Output manager not initialised');
}
const analysisContext = {
section: config.sectionName,
analyzer: config.sectionName,
filePath,
operation: 'genericAnalysis',
};
return await this.executeWithErrorPropagation(async () => {
this.progressReporter.startPhase(config.phase, config.message);
// Handle dependencies with error propagation
const dependencies = [];
if (config.dependencies) {
for (let i = 0; i < config.dependencies.length; i++) {
const depName = config.dependencies[i];
const progressPercent = ((i + 1) / (config.dependencies.length + 2)) * 100;
this.progressReporter.updateProgress({
phase: 'prerequisites',
progress: progressPercent,
message: `Running prerequisite ${depName} analysis...`,
timeElapsed: Date.now() - startTime,
});
let depResult = this.dependencyCache.get(depName);
if (!depResult) {
// Execute dependency with retry logic for transient failures
depResult = await retry_1.RetryUtils.retryAnalysis(() => this.executeDependency(depName, filePath, options), { ...analysisContext, operation: `dependency_${depName}` });
this.dependencyCache.set(depName, depResult);
}
dependencies.push(depResult);
}
}
// Check memory before main analysis
memory_manager_1.globalMemoryManager.checkMemoryUsage(analysisContext);
// Run the main analysis with memory monitoring
const finalProgressPercent = config.dependencies ? 80 : 50;
this.progressReporter.updateProgress({
phase: config.sectionName,
progress: finalProgressPercent,
message: `Running ${config.sectionName} analysis...`,
timeElapsed: Date.now() - startTime,
});
// Check cache first if performance manager is available
let result = null;
const cacheKey = config.sectionName.toLowerCase().replace(/\s+/g, '');
if (this.performanceManager && this.performanceManager.isCachingEnabled()) {
result = await this.performanceManager.getCachedResult(filePath, cacheKey);
if (result) {
logger_1.logger.debug(`Cache hit for ${config.sectionName}`);
this.progressReporter.updateProgress({
phase: config.sectionName,
progress: 90,
message: `Using cached ${config.sectionName} results...`,
timeElapsed: Date.now() - startTime,
});
}
}
// Execute main analysis with retry logic if not cached
if (!result) {
result = await retry_1.RetryUtils.retryAnalysis(() => config.analyzerFactory(filePath, options, dependencies), analysisContext);
// Cache the result if performance manager is available
if (this.performanceManager && this.performanceManager.isCachingEnabled()) {
try {
await this.performanceManager.setCachedResult(filePath, cacheKey, result);
logger_1.logger.debug(`Cached result for ${config.sectionName}`);
}
catch (cacheError) {
logger_1.logger.warn(`Failed to cache result for ${config.sectionName}`);
}
}
}
const processingTime = Date.now() - startTime;
this.progressReporter.completePhase(`${config.sectionName} analysis completed`, processingTime);
// Generate report if formatter is provided
let report = null;
if (config.formatterMethod) {
try {
report = config.formatterMethod(result);
}
catch (formatError) {
logger_1.logger.warn(`Failed to format ${config.sectionName} report, using raw data`, analysisContext, formatError);
report = JSON.stringify(result, null, 2); // Fallback to JSON
}
}
// Generate output files with error handling
let outputFiles = [];
try {
outputFiles = config.outputMethod(this.outputManager, report, result, filePath.split('/').pop());
}
catch (outputError) {
logger_1.logger.warn(`Failed to generate ${config.sectionName} output files`, analysisContext, outputError);
// Continue without output files in case of output errors
}
// Calculate stats with safe property access
const rowsProcessed = Number(error_handler_1.ErrorUtils.safeGet(result, 'overview.structuralDimensions.totalDataRows') ||
error_handler_1.ErrorUtils.safeGet(result, 'performanceMetrics.rowsAnalyzed') ||
error_handler_1.ErrorUtils.safeGet(result, 'stats.rowsProcessed') ||
0);
const warnings = Array.isArray(result.warnings) ? result.warnings.length : 0;
const errors = this.sectionErrors.get(config.sectionName)?.length || 0;
// Log memory usage after analysis
logger_1.logger.memory(analysisContext);
// Show summary
this.progressReporter.showSummary({
processingTime,
rowsProcessed,
warnings,
errors,
});
return {
success: true,
exitCode: 0,
outputFiles,
stats: {
processingTime,
rowsProcessed,
warnings,
errors,
},
};
}, config.sectionName, undefined, // dependencies are handled internally
analysisContext).catch((error) => {
this.progressReporter.errorPhase(`${config.sectionName} analysis failed`);
// Record section-specific errors
const sectionErrors = this.sectionErrors.get(config.sectionName) || [];
if (error instanceof types_2.DataPilotError) {
sectionErrors.push(error);
}
this.sectionErrors.set(config.sectionName, sectionErrors);
throw error;
});
}
/**
* Determine if universal analyzer should be used based on file extension
*/
shouldUseUniversalAnalyzer(filePath) {
const extension = filePath.toLowerCase().split('.').pop();
// Use universal analyzer for non-CSV formats
return ['json', 'jsonl', 'ndjson', 'xlsx', 'xls', 'xlsm', 'tsv', 'tab'].includes(extension || '');
}
/**
* Execute command using universal analyzer for multi-format support
*/
async executeUniversalCommand(command, filePath, options, startTime) {
try {
const analyzer = new universal_analyzer_1.UniversalAnalyzer();
this.progressReporter.startPhase('universal-analysis', `Starting ${command} analysis with multi-format support...`);
// Run universal analysis
const result = await analyzer.analyzeFile(filePath, { ...options, command });
const processingTime = Date.now() - startTime;
this.progressReporter.completePhase('Universal analysis completed', processingTime);
// Convert universal result to CLI format
if (result.success) {
// Generate appropriate output files based on command
const outputFiles = this.generateUniversalOutput(result, command, filePath, options);
this.progressReporter.showSummary({
processingTime,
rowsProcessed: result.metadata?.parserStats?.rowsProcessed || 0,
warnings: 0, // Universal analyzer handles warnings internally
errors: 0,
});
return {
success: true,
exitCode: 0,
outputFiles,
stats: {
processingTime,
rowsProcessed: result.metadata?.parserStats?.rowsProcessed || 0,
warnings: 0,
errors: 0,
},
};
}
else {
throw new Error(result.error || 'Universal analysis failed');
}
}
catch (error) {
this.progressReporter.errorPhase('Universal analysis failed');
throw error;
}
}
/**
* Generate output files for universal analysis results
*/
generateUniversalOutput(result, command, filePath, options) {
if (!this.outputManager) {
return [];
}
const fileName = filePath
.split('/')
.pop()
?.replace(/\.[^/.]+$/, '') || 'analysis';
const outputFiles = [];
try {
// Generate format-agnostic output
const format = result.metadata?.originalFormat || 'unknown';
const analysisData = result.data;
// Create markdown report
const report = this.formatUniversalReport(analysisData, command, format, fileName);
const reportFile = `${fileName}_${command}_${format}_analysis.md`;
this.writeToFile(reportFile, report);
outputFiles.push(reportFile);
// Generate JSON output if requested
if (options.output === 'json') {
const jsonFile = `${fileName}_${command}_${format}_analysis.json`;
this.writeToFile(jsonFile, JSON.stringify(result, null, 2));
outputFiles.push(jsonFile);
}
return outputFiles;
}
catch (error) {
logger_1.logger.warn('Failed to generate universal output files', { error: error.message });
return [];
}
}
/**
* Format universal analysis report
*/
formatUniversalReport(analysisData, command, format, fileName) {
const timestamp = new Date().toISOString();
let report = `# š¤ DataPilot Multi-Format Analysis Report\n\n`;
report += `**File**: ${fileName}\n`;
report += `**Format**: ${format.toUpperCase()}\n`;
report += `**Analysis Type**: ${command}\n`;
report += `**Generated**: ${timestamp}\n`;
report += `**DataPilot Version**: 1.2.1 (Multi-Format Edition)\n\n`;
report += `---\n\n`;
// Add format-specific information
report += `## š Format Analysis\n\n`;
report += `DataPilot successfully detected and processed your **${format.toUpperCase()}** file using the universal parser system. `;
report += `The analysis pipeline was automatically adapted to handle the specific characteristics of this format.\n\n`;
// Add analysis results based on command
if (analysisData) {
report += `## š Analysis Results\n\n`;
// Format the results based on available sections
Object.keys(analysisData).forEach((sectionKey) => {
const sectionData = analysisData[sectionKey];
if (sectionData && typeof sectionData === 'object') {
report += `### ${sectionKey.replace(/([A-Z])/g, ' $1').replace(/^./, (str) => str.toUpperCase())}\n\n`;
// Basic formatting of section data
if (sectionData.summary) {
report += `${sectionData.summary}\n\n`;
}
else {
report += `Analysis completed successfully. Detailed results available in JSON format.\n\n`;
}
}
});
}
report += `---\n\n`;
report += `## šÆ Multi-Format Support\n\n`;
report += `This analysis was performed using DataPilot's universal parser system, which supports:\n`;
report += `- **CSV**: Comma-separated values with auto-detection\n`;
report += `- **TSV**: Tab-separated values\n`;
report += `- **JSON**: Single objects, arrays, and JSON Lines (JSONL)\n`;
report += `- **Excel**: .xlsx, .xls, and .xlsm formats\n\n`;
report += `The same comprehensive statistical analysis is available regardless of your data format.\n\n`;
report += `**Generated by DataPilot v1.2.1** - Universal Data Analysis Engine\n`;
return report;
}
/**
* Execute join analysis for multiple files (used by engineering command)
*/
async executeJoinAnalysisOriginal(filePaths, options, startTime) {
try {
logger_1.logger.info(`Starting join analysis for ${filePaths.length} files`);
// Create join analyzer with options
const { analyze, format } = (0, joins_1.createJoinAnalyzer)({
maxTables: Math.min(filePaths.length, 10), // Keep it reasonable
confidenceThreshold: options.confidence ?? 0.7,
enableFuzzyMatching: true,
enableSemanticAnalysis: true
});
// Run join analysis
const result = await analyze(filePaths);
// Format output
const outputFormat = options.output === 'json' ? 'json' : 'markdown';
const formattedOutput = format(result, outputFormat);
// Output results (simplified for Phase 1)
if (options.outputFile) {
// Write to file if specified
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
fs.writeFileSync(options.outputFile, formattedOutput);
logger_1.logger.info(`Output written to ${options.outputFile}`);
}
else {
// Output to console
console.log(formattedOutput);
}
const duration = Date.now() - startTime;
logger_1.logger.info(`Join analysis completed in ${duration}ms`);
return {
success: true,
exitCode: 0,
message: `Join analysis completed for ${filePaths.length} files - found ${result.candidates.length} join candidates`,
output: formattedOutput
};
}
catch (error) {
logger_1.logger.error('Join analysis failed: ' + error.message);
return this.handleError(error);
}
}
/**
* Enhanced Engineering Analysis - naturally handles single or multiple files
* Single file: Normal Section 5 engineering analysis
* Multiple files: Section 5 analysis + simple relationship analysis
*/
async executeEnhancedEngineering(filePaths, options, startTime) {
const primaryFile = filePaths[0];
try {
// Always run normal Section 5 analysis on the primary file
const section5Result = await this.executeGenericAnalysis({
sectionName: 'Section 5',
phase: 'engineering',
message: 'Starting data engineering analysis...',
dependencies: ['section1', 'section2', 'section3'],
analyzerFactory: async (filePath, options, dependencies) => {
const [section1Data, section2Data, section3Data] = dependencies;
const analyzer = new engineering_1.Section5Analyzer({
targetDatabaseSystem: options.database ||
'postgresql',
mlFrameworkTarget: options.framework ||
'scikit_learn',
});
return await analyzer.analyze(section1Data, section2Data, section3Data);
},
formatterMethod: (result) => engineering_1.Section5Formatter.formatMarkdown(result),
outputMethod: (outputManager, report, result, fileName) => outputManager.outputSection5(report, result, fileName),
}, primaryFile, options, startTime);
// If multiple files provided, add simple relationship analysis
if (filePaths.length > 1) {
logger_1.logger.info(`Multiple files detected - analyzing relationships between ${filePaths.length} files`);
// Get simple column information for each file using Universal Analyzer
const fileSchemas = await this.extractSimpleSchemas(filePaths);
// Simple relationship detection
const relationships = this.detectSimpleRelationships(fileSchemas);
// Append relationship analysis to output
const relationshipReport = this.formatSimpleRelationships(relationships, filePaths);
// Enhanced output that includes both Section 5 and relationships
console.log('\n' + '='.repeat(80));
console.log('š CROSS-FILE RELATIONSHIP ANALYSIS');
console.log('='.repeat(80));
console.log(relationshipReport);
return {
...section5Result,
message: `Engineering analysis completed with relationship analysis for ${filePaths.length} files`,
};
}
return section5Result;
}
catch (error) {
logger_1.logger.error('Enhanced engineering analysis failed: ' + error.message);
return this.handleError(error);
}
}
/**
* Extract simple column schemas from multiple files
*/
async extractSimpleSchemas(filePaths) {
const schemas = [];
for (const filePath of filePaths) {
try {
// Use existing CSV parser to get headers
const parser = new csv_parser_1.CSVParser({
autoDetect: true,
maxRows: 5, // Just need headers + a few rows
trimFields: true,
});
const rows = await parser.parseFile(filePath);
if (rows.length >= 2) {
// Take first 2 rows (header + data row)
rows.splice(2);
}
if (rows.length > 0) {
const columns = rows[0].data;
schemas.push({
fileName: filePath.split('/').pop() || filePath,
columns: columns
});
}
}
catch (error) {
logger_1.logger.warn(`Could not analyze schema for ${filePath}: ${error}`);
}
}
return schemas;
}
/**
* Simple relationship detection between files
*/
detectSimpleRelationships(schemas) {
const relationships = [];
for (let i = 0; i < schemas.length; i++) {
for (let j = i + 1; j < schemas.length; j++) {
const schema1 = schemas[i];
const schema2 = schemas[j];
// Check for potential joins between each pair of columns
for (const col1 of schema1.columns) {
for (const col2 of schema2.columns) {
const match = this.calculateColumnSimilarity(col1, col2);
if (match.confidence > 0.6) {
relationships.push({
file1: schema1.fileName,
column1: col1,
file2: schema2.fileName,
column2: col2,
matchType: match.type,
confidence: match.confidence
});
}
}
}
}
}
return relationships.sort((a, b) => b.confidence - a.confidence);
}
/**
* Simple column similarity calculation
*/
calculateColumnSimilarity(col1, col2) {
const name1 = col1.toLowerCase().trim();
const name2 = col2.toLowerCase().trim();
// Exact match
if (name1 === name2) {
return { confidence: 1.0, type: 'exact' };
}
// Common ID patterns
const idPatterns = [
['id', 'id'],
['customer_id', 'customer_id'],
['user_id', 'user_id'],
['order_id', 'order_id'],
['product_id', 'product_id'],
];
for (const [pattern1, pattern2] of idPatterns) {
if (name1.includes(pattern1) && name2.includes(pattern2)) {
return { confidence: 0.9, type: 'id_pattern' };
}
}
// Semantic similarity for common business terms
const semanticPairs = [
['customer_id', 'client_id'],
['user_id', 'customer_id'],
['product_id', 'item_id'],
['order_id', 'transaction_id'],
['email', 'email_address'],
['name', 'full_name'],
['date', 'timestamp'],
];
for (const [term1, term2] of semanticPairs) {
if ((name1.includes(term1) && name2.includes(term2)) ||
(name1.includes(term2) && name2.includes(term1))) {
return { confidence: 0.8, type: 'semantic' };
}
}
// Partial match (common substring)
if (name1.length > 3 && name2.length > 3) {
const shorter = name1.length < name2.length ? name1 : name2;
const longer = name1.length >= name2.length ? name1 : name2;
if (longer.includes(shorter)) {
return { confidence: 0.7, type: 'partial' };
}
}
return { confidence: 0.0, type: 'none' };
}
/**
* Format simple relationship analysis for output
*/
formatSimpleRelationships(relationships, filePaths) {
if (relationships.length === 0) {
return `\nš **Files Analyzed**: ${filePaths.map(f => f.split('/').pop()).join(', ')}\n\nā **No obvious relationships detected**\n\nThis could mean:\n- Files are independent datasets\n- Relationships use different naming conventions\n- Join columns need data transformation\n\nš” **Recommendation**: Review column names manually for potential joins\n`;
}
let output = `\nš **Files Analyzed**: ${filePaths.map(f => f.split('/').pop()).join(', ')}\n`;
output += `\nā
**Found ${relationships.length} potential join relationship(s)**\n\n`;
// Group by confidence level
const highConfidence = relationships.filter(r => r.confidence >= 0.8);
const mediumConfidence = relationships.filter(r => r.confidence >= 0.6 && r.confidence < 0.8);
if (highConfidence.length > 0) {
output += `### šÆ High Confidence Joins (ā„80%)\n\n`;
for (const rel of highConfidence) {
output += `**${rel.file1}**.${rel.column1} ā **${rel.file2}**.${rel.column2}\n`;
output += `- Confidence: ${(rel.confidence * 100).toFixed(0)}%\n`;
output += `- Type: ${rel.matchType}\n`;
output += `- SQL: \`SELECT * FROM ${rel.file1.replace('.csv', '')} a JOIN ${rel.file2.replace('.csv', '')} b ON a.${rel.column1} = b.${rel.column2}\`\n\n`;
}
}
if (mediumConfidence.length > 0) {
output += `### š¤ Possible Joins (60-79%)\n\n`;
for (const rel of mediumConfidence) {
output += `**${rel.file1}**.${rel.column1} ā **${rel.file2}**.${rel.column2} (${(rel.confidence * 100).toFixed(0)}%)\n`;
}
output += '\nš” **Review these manually** - they might need data transformation\n';
}
return output;
}
/**
* Execute dependency analysis and cache results
*/
async executeDependency(depName, filePath, options) {
switch (depName) {
case 'section1':
const section1Analyzer = new overview_1.Section1Analyzer({
enableFileHashing: options.enableHashing !== false,
includeHostEnvironment: options.includeEnvironment !== false,
privacyMode: options.privacyMode || 'redacted',
detailedProfiling: options.verbose || false,
maxSampleSizeForSparsity: 10000,
enableCompressionAnalysis: options.enableCompressionAnalysis !== false,
enableDataPreview: options.enableDataPreview !== false,
previewRows: options.previewRows || 5,
enableHealthChecks: options.enableHealthChecks !== false,
enableQuickStatistics: options.enableQuickStats !== false,
});
return await section1Analyzer.analyze(filePath, `datapilot ${options.command || 'analysis'} ${filePath}`, []);
case 'section2':
// Parse CSV for Section 2
const parser = new csv_parser_1.CSVParser({
autoDetect: true,
maxRows: options.maxRows || 100000,
trimFields: true,
});
const rows = await parser.parseFile(filePath);
if (rows.length === 0) {
throw new types_3.ValidationError('No data found in file');
}
const hasHeader = parser.getOptions().hasHeader !== false;
const dataStartIndex = hasHeader ? 1 : 0;
const headers = hasHeader && rows.length > 0
? rows[0].data
: rows[0].data.map((_, i) => `Column_${i + 1}`);
const data = rows.slice(dataStartIndex).map((row) => row.data);
const columnTypes = headers.map(() => types_4.DataType.STRING);
const section2Analyzer = new quality_1.Section2Analyzer({
data,
headers,
columnTypes,
rowCount: data.length,
columnCount: headers.length,
config: {
enabledDimensions: ['completeness', 'uniqueness', 'validity'],
strictMode: false,
maxOutlierDetection: 100,
semanticDuplicateThreshold: 0.85,
},
});
return await section2Analyzer.analyze();
case 'section3':
const section3Analyzer = new streaming_analyzer_1.StreamingAnalyzer({
chunkSize: options.chunkSize || 500,
memoryThresholdMB: options.memoryLimit || 100,
maxRowsAnalyzed: options.maxRows || 500000,
enabledAnalyses: ['univariate', 'bivariate', 'correlations'],
significanceLevel: 0.05,
maxCorrelationPairs: 50,
enableMultivariate: true,
});
return await section3Analyzer.analyzeFile(filePath);
default:
throw new Error(`Unknown dependency: ${depName}`);
}
}
/**
* Execute specific CLI command
*/
async executeCommand(command, filePath, options, startTime, args) {
// Initialize performance manager for enhanced performance
if (!this.performanceManager && !options.quiet) {
this.performanceManager = new performance_manager_1.CLIPerformanceManager();
try {
const perfConfig = await this.performanceManager.initialize(filePath, options);
// Use optimized options from performance manager
options = perfConfig.optimizedOptions;
// Start performance monitoring
this.performanceManager.startMonitoring();
if (options.verbose) {
logger_1.logger.info('š Smart performance configuration activated');
if (options.autoConfig) {
this.performanceManager.showDashboard();
}
}
}
catch (error) {
logger_1.logger.warn('Failed to initialize performance manager, continuing with default settings');
this.performanceManager = undefined;
}
}
// Check if universal analyzer should be used (for multi-format support)
const shouldUseUniversal = options.format || this.shouldUseUniversalAnalyzer(filePath);
if (shouldUseUniversal) {
return await this.executeUniversalCommand(command, filePath, options, startTime);
}
// Clear dependency resolver for each command
if (this.dependencyResolver) {
this.dependencyResolver.clear();
}
// Register resolvers for the sections we'll need
this.registerSectionResolvers(filePath, options);
switch (command) {
case 'all':
return await this.executeFullAnalysis(filePath, options, startTime);
case 'overview':
return await this.executeGenericAnalysis({
sectionName: 'Section 1',
phase: 'analysis',
message: 'Starting dataset analysis...',
analyzerFactory: async (filePath, options) => {
const analyzer = new overview_1.Section1Analyzer({
enableFileHashing: options.enableHashing !== false,
includeHostEnvironment: options.includeEnvironment !== false,
privacyMode: options.privacyMode || 'redacted',
detailedProfiling: options.verbose || false,
maxSampleSizeForSparsity: 10000,
enableCompressionAnalysis: options.enableCompressionAnalysis !== false,
enableDataPreview: options.enableDataPreview !== false,
previewRows: options.previewRows || 5,
enableHealthChecks: options.enableHealthChecks !== false,
enableQuickStatistics: options.enableQuickStats !== false,
});
return await analyzer.analyze(filePath, `datapilot ${options.command || 'overview'} ${filePath}`, ['overview']);
},
outputMethod: (outputManager, report, result, fileName) => outputManager.outputSection1(result, fileName),
}, filePath, options, startTime);
case 'quality':
return await this.executeGenericAnalysis({
sectionName: 'Section 2',
phase: 'quality',
message: 'Starting data quality analysis...',
analyzerFactory: async (filePath, options) => {
// Parse CSV to get data for Section 2
const parser = new csv_parser_1.CSVParser({
autoDetect: true,
maxRows: options.maxRows || 100000,
trimFields: true,
});
const rows = await parser.parseFile(filePath);
if (rows.length === 0) {
throw new types_3.ValidationError('No data found in file');
}
const hasHeader = parser.getOptions().hasHeader !== false;
const dataStartIndex = hasHeader ? 1 : 0;
const headers = hasHeader && rows.length > 0
? rows[0].data
: rows[0].data.map((_, i) => `Column_${i + 1}`);
const data = rows.slice(dataStartIndex).map((row) => row.data);
const columnTypes = headers.map(() => types_4.DataType.STRING);
const analyzer = new quality_1.Section2Analyzer({
data,
headers,
columnTypes,
rowCount: data.length,
columnCount: headers.length,
config: {
enabledDimensions: ['completeness', 'uniqueness', 'validity'],
strictMode: false,
maxOutlierDetection: 100,
semanticDuplicateThreshold: 0.85,
},
});
return await analyzer.analyze();
},
formatterMethod: (result) => section2_formatter_1.Section2Formatter.formatReport(result.qualityAudit),
outputMethod: (outputManager, report, result, fileName) => outputManager.outputSection2(result),
}, filePath, options, startTime);
case 'eda':
return await this.executeGenericAnalysis({
sectionName: 'Section 3',
phase: 'eda',
message: 'Starting exploratory data analysis...',
analyzerFactory: async (filePath, options) => {
const analyzer = new streaming_analyzer_1.StreamingAnalyzer({
chunkSize: options.chunkSize || 500,
memoryThresholdMB: options.memoryLimit || 100,
maxRowsAnalyzed: options.maxRows || 500000,
enabledAnalyses: ['univariate', 'bivariate', 'correlations'],
significanceLevel: 0.05,
maxCorrelationPairs: 50,
enableMultivariate: true,
});
return await analyzer.analyzeFile(filePath);
},
formatterMethod: (result) => section3_formatter_1.Section3Formatter.formatSection3(result),
outputMethod: (outputManager, report, result, fileName) => outputManager.outputSection3(report, result, fileName),
}, filePath, options, startTime);
case 'viz':
return await this.executeGenericAnalysis({
sectionName: 'Section 4',
phase: 'section4',
message: 'Starting visualization intelligence analysis...',
dependencies: ['section1', 'section3'],
analyzerFactory: async (filePath, options, dependencies) => {
const [section1Data, section3Data] = dependencies;
const analyzer = new visualization_1.Section4Analyzer({
accessibilityLevel: options.accessibility || types_1.AccessibilityLevel.GOOD,
complexityThreshold: options.complexity || types_1.ComplexityLevel.MODERATE,
maxRecommendationsPerChart: options.maxRecommendations || 3,
includeCodeExamples: options.includeCode || false,
enabledRecommendations: [
types_1.RecommendationType.UNIVARIATE,
types_1.RecommendationType.BIVARIATE,
types_1.RecommendationType.DASHBOARD,
types_1.RecommendationType.ACCESSIBILITY,
types_1.RecommendationType.PERFORMANCE,
],
targetLibraries: ['d3', 'plotly', 'observable'],
});
return await analyzer.analyze(section1Data, section3Data);
},
formatterMethod: (result) => visualization_1.Section4Formatter.formatSection4(result),
outputMethod: (outputManager, report, result, fileName) => outputManager.outputSection4(report, result, fileName),
}, filePath, options, startTime);
case 'engineering':
// Enhanced engineering analysis - handles single or multiple files seamlessly
return await this.executeEnhancedEngineering(args, options, startTime);
case 'modeling':
return await this.executeGenericAnalysis({
sectionName: 'Section 6',
phase: 'modeling',
message: 'Starting predictive modeling analysis...',
dependencies: ['section1', 'section2', 'section3'],
analyzerFactory: async (filePath, options, dependencies) => {
const [section1Data, section2Data, section3Data] = dependencies;
// Need Section 5 result as well
const section5Analyzer = new engineering_1.Section5Analyzer({
targetDatabaseSystem: options.database ||
'postgresql',
mlFrameworkTarget: options.framework ||
'scikit_learn',
});
const section5Result = await section5Analyzer.analyze(section1Data, section2Data, section3Data);
const analyzer = new modeling_1.Section6Analyzer({
focusAreas: options.focus || [
'regression',
'binary_classification',
'clustering',
],
complexityPreference: options.complexity || 'moderate',
interpretabilityRequirement