datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
472 lines • 19.9 kB
JavaScript
;
/**
* Universal Analyzer - Multi-format data analysis orchestrator
* Integrates with the existing 6-section analysis pipeline
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.UniversalAnalyzer = void 0;
const parser_registry_1 = require("../parsers/base/parser-registry");
const json_parser_1 = require("../parsers/json-parser");
const csv_parser_adapter_1 = require("../parsers/adapters/csv-parser-adapter");
const excel_parser_1 = require("../parsers/excel-parser");
const tsv_parser_1 = require("../parsers/tsv-parser");
const parquet_parser_1 = require("../parsers/parquet-parser");
const types_1 = require("../core/types");
const logger_1 = require("../utils/logger");
// Import existing analyzers (these remain unchanged)
const overview_1 = require("../analyzers/overview");
const quality_1 = require("../analyzers/quality");
const streaming_analyzer_1 = require("../analyzers/streaming/streaming-analyzer");
const visualization_1 = require("../analyzers/visualization");
const engineering_1 = require("../analyzers/engineering");
const modeling_1 = require("../analyzers/modeling");
/**
* Universal analyzer that works with any supported data format
*/
class UniversalAnalyzer {
registry;
initialized = false;
constructor() {
this.registry = parser_registry_1.globalParserRegistry;
}
/**
* Initialize the analyzer with all available parsers
*/
initializeParsers() {
if (this.initialized)
return;
// Register CSV parser (highest priority for backwards compatibility)
this.registry.register({
format: 'csv',
parserFactory: (options) => (0, csv_parser_adapter_1.createCSVParserAdapter)(options),
detector: {
detect: async (filePath) => {
const adapter = (0, csv_parser_adapter_1.createCSVParserAdapter)();
return adapter.detect(filePath);
},
getSupportedExtensions: () => ['.csv'],
getFormatName: () => 'csv',
},
priority: 100,
extensions: ['.csv'],
});
// Register TSV parser
this.registry.register({
format: 'tsv',
parserFactory: (options) => (0, tsv_parser_1.createTSVParser)(options),
detector: new tsv_parser_1.TSVDetector(),
priority: 90,
extensions: ['.tsv', '.tab'],
});
// Register JSON parser
this.registry.register({
format: 'json',
parserFactory: (options) => (0, json_parser_1.createJSONParser)(options),
detector: new json_parser_1.JSONDetector(),
priority: 80,
extensions: ['.json', '.jsonl', '.ndjson'],
});
// Register Excel parser
this.registry.register({
format: 'excel',
parserFactory: (options) => (0, excel_parser_1.createExcelParser)(options),
detector: new excel_parser_1.ExcelDetector(),
priority: 70,
extensions: ['.xlsx', '.xls', '.xlsm'],
});
// Register Parquet parser
this.registry.register({
format: 'parquet',
parserFactory: (options) => (0, parquet_parser_1.createParquetParser)(options),
detector: new parquet_parser_1.ParquetDetector(),
priority: 60,
extensions: ['.parquet'],
});
this.initialized = true;
logger_1.logger.info(`Initialized universal analyzer with ${this.registry.getSupportedFormats().length} formats`);
}
/**
* Analyze any supported file format
*/
async analyzeFile(filePath, options) {
this.initializeParsers();
try {
// 1. Auto-detect format and get parser
logger_1.logger.info(`Starting universal analysis for: ${filePath}`);
// Convert CLIOptions to ParseOptions
const parseOptions = {
maxRows: options.maxRows,
encoding: options.encoding,
format: options.format,
delimiter: options.delimiter,
quote: options.quote,
hasHeader: options.hasHeader,
jsonPath: options.jsonPath,
arrayMode: options.arrayMode,
flattenObjects: options.flattenObjects,
sheetName: options.sheetName,
sheetIndex: options.sheetIndex,
columns: options.columns,
rowStart: options.rowStart,
rowEnd: options.rowEnd,
};
const { parser, format, detection } = await this.registry.getParser(filePath, parseOptions);
logger_1.logger.info(`Detected format: ${format} (confidence: ${(detection.confidence * 100).toFixed(1)}%)`);
// 2. Validate file can be parsed
const validation = await parser.validate(filePath);
if (!validation.canProceed) {
throw new types_1.DataPilotError(`Cannot parse file: ${validation.errors.join(', ')}`, 'UNIVERSAL_PARSE_ERROR', types_1.ErrorSeverity.HIGH, types_1.ErrorCategory.VALIDATION);
}
if (validation.warnings.length > 0) {
logger_1.logger.warn(`Parsing warnings: ${validation.warnings.join(', ')}`);
}
// 3. Convert to common dataset format
const dataset = await this.parseToDataset(parser, filePath, parseOptions, format, detection);
// 4. Run the same 6-section analysis pipeline
const analysisResult = await this.runAnalysisPipeline(dataset, options);
return {
success: true,
exitCode: 0,
data: analysisResult,
format: options.format || 'markdown',
metadata: {
command: options.command || 'all',
filePath,
originalFormat: format,
detection: {
format,
confidence: detection.confidence,
metadata: detection.metadata,
},
parserStats: parser.getStats(),
timestamp: new Date().toISOString(),
version: '1.2.1', // Multi-format support version
},
};
}
catch (error) {
return this.handleAnalysisError(error, filePath);
}
}
/**
* Parse file using detected parser and convert to universal dataset format
*/
async parseToDataset(parser, filePath, options, format, detection) {
const rows = [];
let headers = [];
let hasHeader = false;
logger_1.logger.info('Parsing file to dataset format...');
// Parse file and collect rows
for await (const row of parser.parse(filePath, {
maxRows: options.maxRows,
hasHeader: options.hasHeader,
encoding: options.encoding,
delimiter: options.delimiter,
quote: options.quote,
jsonPath: options.jsonPath,
arrayMode: options.arrayMode,
flattenObjects: options.flattenObjects,
sheetName: options.sheetName,
sheetIndex: options.sheetIndex,
columns: options.columns,
rowStart: options.rowStart,
rowEnd: options.rowEnd,
})) {
if (row.index === 0 && !hasHeader) {
// First row - determine if it's headers or data
const isHeaderRow = this.detectHeaderRow(row.data, format);
if (isHeaderRow) {
headers = row.data;
hasHeader = true;
continue;
}
else {
// Generate column names
headers = row.data.map((_, i) => `column_${i + 1}`);
}
}
rows.push(row.data);
}
const stats = parser.getStats();
logger_1.logger.info(`Parsed ${rows.length} rows with ${headers.length} columns`);
return {
headers,
rows,
metadata: {
format,
filePath,
totalRows: rows.length,
parserStats: stats,
detection,
},
};
}
/**
* Detect if first row contains headers
*/
detectHeaderRow(row, format) {
// Format-specific header detection logic
switch (format) {
case 'json':
// JSON usually has meaningful keys as headers
return true;
case 'parquet':
// Parquet always has schema-defined column names
return true;
case 'csv':
// CSV header detection (existing logic)
return this.detectCSVHeaders(row);
default:
// Default: assume first row is header if it contains non-numeric values
return row.some((cell) => isNaN(Number(cell)) && cell.trim() !== '');
}
}
detectCSVHeaders(row) {
// Simple heuristic: if more than half the cells are non-numeric, likely headers
const nonNumeric = row.filter((cell) => {
const trimmed = cell.trim();
return trimmed !== '' && isNaN(Number(trimmed));
});
return nonNumeric.length > row.length / 2;
}
/**
* Run the existing 6-section analysis pipeline on the universal dataset
*/
async runAnalysisPipeline(dataset, options) {
const results = {};
// Section 1: Overview Analysis (adapted for universal format)
if (this.shouldRunSection(1, options)) {
logger_1.logger.info('Running Section 1: Overview Analysis');
results.section1 = await this.runSection1Analysis(dataset, options);
}
// Section 2: Data Quality Analysis
if (this.shouldRunSection(2, options)) {
logger_1.logger.info('Running Section 2: Data Quality Analysis');
results.section2 = await this.runSection2Analysis(dataset, options);
}
// Section 3: Exploratory Data Analysis
if (this.shouldRunSection(3, options)) {
logger_1.logger.info('Running Section 3: EDA');
results.section3 = await this.runSection3Analysis(dataset, options);
}
// Section 4: Visualization Intelligence
if (this.shouldRunSection(4, options)) {
logger_1.logger.info('Running Section 4: Visualization Intelligence');
results.section4 = await this.runSection4Analysis(dataset, options);
}
// Section 5: Data Engineering
if (this.shouldRunSection(5, options)) {
logger_1.logger.info('Running Section 5: Data Engineering');
results.section5 = await this.runSection5Analysis(dataset, options);
}
// Section 6: Modeling Strategy
if (this.shouldRunSection(6, options)) {
logger_1.logger.info('Running Section 6: Modeling Strategy');
results.section6 = await this.runSection6Analysis(dataset, options);
}
return results;
}
shouldRunSection(sectionNumber, options) {
if (options.sections) {
return options.sections.includes(sectionNumber.toString());
}
// Default sections based on command
switch (options.command) {
case 'overview':
return sectionNumber === 1;
case 'quality':
return sectionNumber === 2;
case 'eda':
return sectionNumber === 3;
case 'visualization':
return sectionNumber === 4;
case 'engineering':
return sectionNumber === 5;
case 'modeling':
return sectionNumber === 6;
case 'all':
return true;
default:
return true;
}
}
// Section analysis methods (these adapt existing analyzers to work with universal dataset)
async runSection1Analysis(dataset, options) {
const analyzer = new overview_1.Section1Analyzer({
enableFileHashing: options.enableHashing !== false,
includeHostEnvironment: options.includeEnvironment !== false,
privacyMode: options.privacyMode || 'redacted',
detailedProfiling: options.verbose || false,
maxSampleSizeForSparsity: 10000,
enableCompressionAnalysis: options.enableCompressionAnalysis !== false,
enableDataPreview: options.enableDataPreview !== false,
previewRows: options.previewRows || 5,
enableHealthChecks: options.enableHealthChecks !== false,
enableQuickStatistics: options.enableQuickStats !== false,
});
// Section1 expects filePath, command, and analysis sections
return analyzer.analyze(dataset.metadata.filePath, `datapilot ${options.command || 'analysis'} ${dataset.metadata.filePath}`, []);
}
async runSection2Analysis(dataset, options) {
const analyzer = new quality_1.Section2Analyzer({
data: dataset.rows,
headers: dataset.headers,
columnTypes: dataset.headers.map(() => 'string'),
rowCount: dataset.rows.length,
columnCount: dataset.headers.length,
config: {
enabledDimensions: ['completeness', 'uniqueness', 'validity'],
strictMode: false,
maxOutlierDetection: 100,
semanticDuplicateThreshold: 0.85,
},
});
return analyzer.analyze();
}
async runSection3Analysis(dataset, options) {
const analyzer = new streaming_analyzer_1.StreamingAnalyzer({
chunkSize: options.chunkSize || 500,
memoryThresholdMB: options.memoryLimit || 100,
maxRowsAnalyzed: options.maxRows || 500000,
enabledAnalyses: ['univariate', 'bivariate', 'correlations'],
significanceLevel: 0.05,
maxCorrelationPairs: 50,
enableMultivariate: true,
});
// Section3 expects a file path, not data stream
return analyzer.analyzeFile(dataset.metadata.filePath);
}
async runSection4Analysis(dataset, options) {
const analyzer = new visualization_1.Section4Analyzer({
accessibilityLevel: options.accessibility || 'good',
complexityThreshold: options.complexity || 'moderate',
maxRecommendationsPerChart: options.maxRecommendations || 3,
includeCodeExamples: options.includeCode || false,
enabledRecommendations: [
'UNIVARIATE',
'BIVARIATE',
'DASHBOARD',
'ACCESSIBILITY',
'PERFORMANCE',
],
targetLibraries: ['d3', 'plotly', 'observable'],
});
// Section4 needs dependencies from previous sections
// For now, we'll need to create mock dependencies
// This will be replaced with actual dependency resolution
const mockSection1 = {
overview: { structuralDimensions: { totalDataRows: dataset.rows.length } },
};
const mockSection3 = { performanceMetrics: { rowsAnalyzed: dataset.rows.length } };
return analyzer.analyze(mockSection1, mockSection3);
}
async runSection5Analysis(dataset, options) {
const analyzer = new engineering_1.Section5Analyzer({
targetDatabaseSystem: options.database || 'postgresql',
mlFrameworkTarget: options.framework || 'scikit_learn',
});
// Section5 needs dependencies from previous sections
const mockSection1 = {
overview: { structuralDimensions: { totalDataRows: dataset.rows.length } },
};
const mockSection2 = { qualityAudit: { overallScore: 85 } };
const mockSection3 = { performanceMetrics: { rowsAnalyzed: dataset.rows.length } };
return analyzer.analyze(mockSection1, mockSection2, mockSection3);
}
async runSection6Analysis(dataset, options) {
const analyzer = new modeling_1.Section6Analyzer({
focusAreas: options.focus || ['regression', 'binary_classification', 'clustering'],
complexityPreference: options.complexity || 'moderate',
interpretabilityRequirement: options.interpretability || 'medium',
});
// Section6 needs dependencies from previous sections
const mockSection1 = {
overview: { structuralDimensions: { totalDataRows: dataset.rows.length } },
};
const mockSection2 = { qualityAudit: { overallScore: 85 } };
const mockSection3 = { performanceMetrics: { rowsAnalyzed: dataset.rows.length } };
const mockSection5 = { engineeringGuidance: { recommendations: [] } };
return analyzer.analyze(mockSection1, mockSection2, mockSection3, mockSection5);
}
/**
* Create async iterable data stream from dataset
*/
async *createDataStream(dataset) {
for (const row of dataset.rows) {
yield row;
}
}
/**
* Handle analysis errors with helpful suggestions
*/
handleAnalysisError(error, filePath) {
const supportedFormats = this.registry.getSupportedFormats();
const supportedExtensions = this.registry.getSupportedExtensions();
return {
success: false,
exitCode: 1,
error: error.message,
suggestions: [
`Check if file format is supported: ${supportedFormats.join(', ')}`,
`Supported extensions: ${supportedExtensions.join(', ')}`,
'Try specifying format explicitly: --format json',
'Verify file is not corrupted',
'Check file permissions',
'Use --help for more information',
],
metadata: {
filePath,
supportedFormats,
supportedExtensions,
timestamp: new Date().toISOString(),
},
};
}
/**
* Get supported formats for help/error messages
*/
getSupportedFormats() {
this.initializeParsers();
return this.registry.getSupportedFormats();
}
/**
* Validate file format is supported
*/
async validateFile(filePath) {
this.initializeParsers();
try {
const validation = await this.registry.validateFile(filePath);
if (validation.supported && validation.bestMatch) {
return {
supported: true,
format: validation.bestMatch.format,
confidence: validation.bestMatch.detection.confidence,
suggestions: [],
};
}
else {
return {
supported: false,
suggestions: [
`File format not supported or confidence too low`,
`Supported formats: ${this.getSupportedFormats().join(', ')}`,
'Try converting to a supported format',
'Check if file is corrupted',
],
};
}
}
catch (error) {
return {
supported: false,
suggestions: [
`Error validating file: ${error.message}`,
'Check file exists and is readable',
'Verify file format is supported',
],
};
}
}
}
exports.UniversalAnalyzer = UniversalAnalyzer;
//# sourceMappingURL=universal-analyzer.js.map