datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
499 lines (496 loc) • 23.9 kB
JavaScript
"use strict";
/**
* CLI Argument Parser and Validator
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.ArgumentParser = void 0;
const commander_1 = require("commander");
const fs_1 = require("fs");
const path_1 = require("path");
const types_1 = require("./types");
class ArgumentParser {
program;
constructor() {
this.program = new commander_1.Command();
this.setupCommands();
}
/**
* Parse command line arguments
*/
parse(argv) {
try {
// Handle empty arguments case
if (argv.length <= 2) {
return {
command: 'help',
args: [],
options: {},
startTime: Date.now(),
workingDirectory: process.cwd(),
};
}
// Check for help flag to avoid process.exit
if (argv.includes('--help') || argv.includes('-h')) {
return {
command: 'help',
args: [],
options: {},
startTime: Date.now(),
workingDirectory: process.cwd(),
};
}
this.program.parse(argv);
// Get the parsed command and options
const lastContext = this.getLastContext();
if (lastContext) {
return {
...lastContext,
startTime: Date.now(),
workingDirectory: process.cwd(),
};
}
// Fallback for basic parsing
// Check if we have any remaining args after parsing
const remainingArgs = this.program.args;
const command = remainingArgs.length > 0 ? remainingArgs[0] : 'help';
const globalOptions = this.program.opts();
return {
command,
args: this.program.args,
options: this.validateOptions(globalOptions),
startTime: Date.now(),
workingDirectory: process.cwd(),
};
}
catch (error) {
if (error instanceof Error) {
throw new types_1.ValidationError(error.message);
}
throw new types_1.ValidationError('Failed to parse command line arguments');
}
}
/**
* Set up all CLI commands and options
*/
setupCommands() {
this.program
.name('datapilot')
.description('A lightweight CLI statistical computation engine for comprehensive CSV data analysis')
.version('1.4.0')
.helpOption(false) // Disable automatic help to handle it manually
.addHelpText('after', `
Confidence Metrics Guide:
DataPilot reports confidence scores throughout the analysis. These indicate the reliability of automated decisions:
• Parsing Confidence (Section 1): 95% = High certainty in CSV parameter detection
• Type Detection Confidence (Section 3): 0.85+ = Strong evidence for data type classification
• Visualization Confidence (Section 4): 0.9+ = Chart type strongly recommended
• Quality Scores (Section 2): Based on completeness of quality dimension analysis
• ML Readiness (Section 5): Reflects assessment completeness and data suitability
• Modeling Confidence (Section 6): Categorical levels based on task clarity and algorithm fit
Use --verbose for detailed confidence explanations in reports.`);
// Global options
this.program
.option('-v, --verbose', 'Enable verbose output with detailed progress')
.option('-q, --quiet', 'Suppress all output except errors')
.option('--no-progress', 'Disable progress indicators')
.option('--dry-run', 'Validate inputs without performing analysis')
.option('-h, --help', 'Display help information')
// Performance and auto-configuration options
.option('--auto-config', 'Enable smart auto-configuration based on system resources')
.option('--preset <name>', 'Use performance preset (ultra-large-files, large-files, balanced, speed-optimized, memory-constrained)')
.option('--threads <number>', 'Number of worker threads (auto-detected if not specified)', this.parseInteger)
.option('--cache', 'Enable section result caching for performance')
.option('--cache-size <mb>', 'Cache size limit in MB', this.parseInteger)
.option('--no-cache', 'Disable all caching')
.option('--streaming', 'Force streaming optimizations for large files')
.option('--progressive', 'Enable progressive analysis reporting');
// Main command: analyze all sections
this.program
.command('all')
.argument('<file>', 'CSV file to analyze')
.description('Run complete analysis on a CSV file (all sections)')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--delimiter <char>', 'Specify delimiter character (e.g., ";" for semicolon)')
.option('--max-rows <number>', 'Maximum rows to process', this.parseInteger)
.option('--no-hashing', 'Disable file hashing for faster processing')
.option('--no-environment', 'Exclude host environment details from report')
.option('--privacy <mode>', 'Privacy mode (full, redacted, minimal)', 'redacted')
.option('--max-memory <mb>', 'Maximum memory usage in MB', this.parseInteger)
.option('--force', 'Ignore warnings and force processing')
.option('--preview-rows <number>', 'Number of rows to show in data preview (default: 5)', this.parseInteger)
.option('--no-compression-analysis', 'Disable compression analysis')
.option('--no-health-checks', 'Disable file health checks')
.option('--no-quick-stats', 'Disable quick column statistics')
.option('--no-data-preview', 'Disable data preview generation')
.action(this.createCommandHandler('all'));
// Section-specific commands
this.program
.command('overview')
.alias('ove')
.argument('<file>', 'CSV file to analyze')
.description('Generate dataset overview (Section 1 only)')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--delimiter <char>', 'Specify delimiter character (e.g., ";" for semicolon)')
.option('--no-hashing', 'Disable file hashing for faster processing')
.option('--privacy <mode>', 'Privacy mode (full, redacted, minimal)', 'redacted')
.option('--preview-rows <number>', 'Number of rows to show in data preview (default: 5)', this.parseInteger)
.option('--no-compression-analysis', 'Disable compression analysis')
.option('--no-health-checks', 'Disable file health checks')
.option('--no-quick-stats', 'Disable quick column statistics')
.option('--no-data-preview', 'Disable data preview generation')
.action(this.createCommandHandler('overview'));
// Section 2: Quality analysis
this.program
.command('quality')
.alias('qua')
.argument('<file>', 'CSV file to analyze')
.description('Run data quality audit (Section 2)')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--delimiter <char>', 'Specify delimiter character (e.g., ";" for semicolon)')
.option('--max-rows <number>', 'Maximum rows to process', this.parseInteger)
.option('--strict', 'Enable strict quality checking mode')
.action(this.createCommandHandler('quality'));
this.program
.command('eda')
.argument('<file>', 'CSV file to analyze')
.description('Perform exploratory data analysis (Section 3)')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--delimiter <char>', 'Specify delimiter character (e.g., ";" for semicolon)')
.option('--max-rows <number>', 'Maximum rows to analyze', this.parseInteger)
.option('--chunk-size <number>', 'Chunk size for streaming processing', this.parseInteger)
.option('--memory-limit <mb>', 'Memory limit in MB', this.parseInteger)
.action(this.createCommandHandler('eda'));
this.program
.command('visualization')
.alias('vis')
.argument('<file>', 'CSV file to analyze')
.description('Generate visualization recommendations (Section 4)')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--accessibility <level>', 'Accessibility level (excellent, good, adequate)', 'good')
.option('--complexity <level>', 'Complexity threshold (simple, moderate, complex)', 'moderate')
.option('--max-recommendations <number>', 'Maximum recommendations per chart', this.parseInteger)
.option('--include-code', 'Include implementation code examples')
.action(this.createCommandHandler('viz'));
this.program
.command('engineering')
.alias('eng')
.argument('<files...>', 'CSV file(s) to analyze - single file for schema analysis, multiple files for join analysis')
.description('Provide data engineering insights (Section 5) - supports multi-file join analysis')
.option('-o, --output <format>', 'Output format', 'markdown')
.option('--confidence <threshold>', 'Join confidence threshold (0-1)', this.parseFloat, 0.7)
.action(this.createJoinCommandHandler('engineering'));
this.program
.command('modeling')
.alias('mod')
.argument('<file>', 'CSV file to analyze')
.description('Generate predictive modeling guidance (Section 6)')
.option('-o, --output <format>', 'Output format', 'markdown')
.action(this.createCommandHandler('modeling'));
// Join Analysis Commands
this.program
.command('join')
.argument('<files...>', 'CSV files to analyze for join relationships')
.description('Analyze join relationships between multiple CSV files')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--confidence <threshold>', 'Minimum confidence threshold (0-1)', this.parseFloat, 0.7)
.option('--max-tables <number>', 'Maximum number of tables to analyze', this.parseInteger, 10)
.option('--enable-fuzzy', 'Enable fuzzy matching for column names')
.option('--enable-semantic', 'Enable semantic analysis for relationships')
.option('--enable-temporal', 'Enable temporal join detection')
.option('--performance-mode <mode>', 'Analysis mode (fast, balanced, thorough)', 'balanced')
.action(this.createJoinCommandHandler('join'));
this.program
.command('discover')
.argument('<directory>', 'Directory containing CSV files to discover relationships')
.description('Auto-discover join relationships in a directory of CSV files')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--pattern <glob>', 'File pattern to match (e.g., "*.csv")', '*.csv')
.option('--confidence <threshold>', 'Minimum confidence threshold (0-1)', this.parseFloat, 0.7)
.option('--recursive', 'Search subdirectories recursively')
.action(this.createJoinCommandHandler('discover'));
this.program
.command('join-wizard')
.argument('<file1>', 'First CSV file')
.argument('<file2>', 'Second CSV file')
.description('Interactive join wizard for two CSV files')
.option('--on <columns>', 'Specific columns to join on (comma-separated)')
.option('--preview-rows <number>', 'Number of rows to preview', this.parseInteger, 5)
.action(this.createJoinCommandHandler('join-wizard'));
this.program
.command('optimize-joins')
.argument('<files...>', 'CSV files to optimize for joining')
.description('Analyze and recommend join optimizations')
.option('-o, --output <format>', 'Output format (txt, markdown, json, yaml)', 'markdown')
.option('--output-file <file>', 'Write output to file instead of stdout')
.option('--include-sql', 'Generate SQL optimization suggestions')
.option('--include-indexing', 'Include indexing recommendations')
.action(this.createJoinCommandHandler('optimize-joins'));
// Utility commands
this.program
.command('validate')
.argument('<file>', 'CSV file to validate')
.description('Validate CSV file format without full analysis')
.option('--encoding <encoding>', 'Expected encoding (utf8, utf16le, etc.)')
.option('--delimiter <char>', 'Expected delimiter character')
.action(this.createCommandHandler('validate'));
this.program
.command('info')
.argument('<file>', 'CSV file to inspect')
.description('Show quick file information and format detection')
.action(this.createCommandHandler('info'));
// Performance and diagnostics commands
this.program
.command('perf')
.description('Show performance dashboard and system information')
.option('--cache-stats', 'Show detailed cache statistics')
.action(this.createNoFileCommandHandler('perf'));
this.program
.command('clear-cache')
.description('Clear all cached analysis results')
.action(this.createNoFileCommandHandler('clear-cache'));
}
/**
* Create command handler that stores context for later execution
*/
createCommandHandler(commandName) {
return (file, options) => {
// Store the context for the main CLI to pick up
this.program._lastContext = {
command: commandName,
file,
options,
args: [file], // Include the file in args for compatibility
};
};
}
/**
* Create join command handler for multi-file operations
*/
createJoinCommandHandler(commandName) {
return (filesOrDirectory, options) => {
// Store the context for the main CLI to pick up
this.program._lastContext = {
command: commandName,
file: Array.isArray(filesOrDirectory) ? filesOrDirectory[0] : filesOrDirectory,
options,
args: Array.isArray(filesOrDirectory) ? filesOrDirectory : [filesOrDirectory],
};
};
}
/**
* Create command handler for commands that don't require a file argument
*/
createNoFileCommandHandler(commandName) {
return (options) => {
// Store the context for the main CLI to pick up
this.program._lastContext = {
command: commandName,
file: '', // No file required
options,
args: [],
};
};
}
/**
* Create stub handler for unimplemented sections
*/
createStubHandler(section, sectionName) {
return (file) => {
console.log(`\n🚧 ${sectionName} (Section ${this.getSectionNumber(section)}) - Coming Soon!`);
console.log(`The ${section} command will be available in a future release.`);
console.log(`\nFor now, use 'datapilot overview ${file}' to analyze your dataset.`);
process.exit(0);
};
}
/**
* Get section number for display
*/
getSectionNumber(section) {
const numbers = {
overview: '1',
quality: '2',
eda: '3',
viz: '4',
engineering: '5',
modeling: '6',
};
return numbers[section] || '?';
}
/**
* Validate and normalize CLI options
*/
validateOptions(rawOptions) {
const options = {};
// Output options
if (rawOptions.output) {
if (!['txt', 'markdown', 'json', 'yaml'].includes(rawOptions.output)) {
throw new types_1.ValidationError('Output format must be one of: txt, markdown, json, yaml');
}
options.output = rawOptions.output;
}
if (rawOptions.outputFile) {
options.outputFile = (0, path_1.resolve)(rawOptions.outputFile);
}
// Verbosity options
options.verbose = Boolean(rawOptions.verbose);
options.quiet = Boolean(rawOptions.quiet);
if (options.verbose && options.quiet) {
throw new types_1.ValidationError('Cannot use both --verbose and --quiet options');
}
// Analysis options
if (rawOptions.maxRows !== undefined) {
const maxRows = Number(rawOptions.maxRows);
if (maxRows <= 0) {
throw new types_1.ValidationError('Max rows must be a positive number');
}
options.maxRows = maxRows;
}
// Delimiter validation
if (rawOptions.delimiter !== undefined) {
const delimiter = rawOptions.delimiter;
if (delimiter.length !== 1) {
throw new types_1.ValidationError('Delimiter must be a single character');
}
options.delimiter = delimiter;
}
options.enableHashing = !rawOptions.noHashing;
options.includeEnvironment = !rawOptions.noEnvironment;
if (rawOptions.privacy) {
if (!['full', 'redacted', 'minimal'].includes(rawOptions.privacy)) {
throw new types_1.ValidationError('Privacy mode must be one of: full, redacted, minimal');
}
options.privacyMode = rawOptions.privacy;
}
// Performance options
if (rawOptions.maxMemory !== undefined) {
const maxMemory = Number(rawOptions.maxMemory);
if (maxMemory <= 0) {
throw new types_1.ValidationError('Max memory must be a positive number');
}
options.maxMemory = maxMemory;
}
// Auto-configuration options
options.autoConfig = Boolean(rawOptions.autoConfig);
if (rawOptions.preset) {
const validPresets = ['ultra-large-files', 'large-files', 'balanced', 'speed-optimized', 'memory-constrained'];
if (!validPresets.includes(rawOptions.preset)) {
throw new types_1.ValidationError(`Performance preset must be one of: ${validPresets.join(', ')}`);
}
options.preset = rawOptions.preset;
}
if (rawOptions.threads !== undefined) {
const threads = Number(rawOptions.threads);
if (threads <= 0) {
throw new types_1.ValidationError('Thread count must be a positive number');
}
options.threads = threads;
}
// Caching options
options.enableCaching = rawOptions.cache ? true : rawOptions.noCache ? false : undefined;
if (rawOptions.cacheSize !== undefined) {
const cacheSize = Number(rawOptions.cacheSize);
if (cacheSize <= 0) {
throw new types_1.ValidationError('Cache size must be a positive number');
}
options.cacheSize = cacheSize;
}
// Performance optimization options
options.streamingOptimizations = Boolean(rawOptions.streaming);
options.progressiveReporting = Boolean(rawOptions.progressive);
// Behaviour options
options.force = Boolean(rawOptions.force);
options.dryRun = Boolean(rawOptions.dryRun);
options.showProgress = !rawOptions.noProgress && !options.quiet;
return options;
}
/**
* Validate file path and accessibility
*/
validateFile(filePath) {
const resolvedPath = (0, path_1.resolve)(filePath);
if (!(0, fs_1.existsSync)(resolvedPath)) {
throw new types_1.FileError(`File not found: ${filePath}`, resolvedPath);
}
try {
const stats = (0, fs_1.statSync)(resolvedPath);
if (!stats.isFile()) {
throw new types_1.FileError(`Path is not a file: ${filePath}`, resolvedPath);
}
if (stats.size === 0) {
throw new types_1.FileError(`File is empty: ${filePath}`, resolvedPath);
}
if (stats.size > 10 * 1024 * 1024 * 1024) {
// 10GB
throw new types_1.FileError(`File is too large (>10GB): ${filePath}`, resolvedPath);
}
}
catch (error) {
if (error instanceof types_1.FileError) {
throw error;
}
throw new types_1.FileError(`Cannot access file: ${filePath}`, resolvedPath);
}
return resolvedPath;
}
/**
* Parse integer with validation
*/
parseInteger(value) {
const parsed = parseInt(value, 10);
if (isNaN(parsed)) {
throw new types_1.ValidationError(`Invalid number: ${value}`);
}
return parsed;
}
parseFloat(value) {
const parsed = parseFloat(value);
if (isNaN(parsed)) {
throw new types_1.ValidationError(`Invalid number: ${value}`);
}
return parsed;
}
/**
* Get the stored command context (used by main CLI)
*/
getLastContext() {
const context = this.program._lastContext;
if (context) {
// Merge global options with command-specific options
const globalOptions = this.program.opts();
const mergedOptions = { ...globalOptions, ...context.options };
return {
...context,
options: this.validateOptions(mergedOptions),
};
}
return context;
}
/**
* Show help for specific command or general help
*/
showHelp(command) {
if (command) {
const cmd = this.program.commands.find((c) => c.name() === command);
if (cmd) {
// Use outputHelp instead of help to avoid process.exit
process.stdout.write(cmd.helpInformation());
}
else {
console.error(`Unknown command: ${command}`);
process.stdout.write(this.program.helpInformation());
}
}
else {
process.stdout.write(this.program.helpInformation());
}
}
}
exports.ArgumentParser = ArgumentParser;
//# sourceMappingURL=argument-parser.js.map