datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
248 lines • 9.2 kB
JavaScript
;
/**
* CSV Parser Adapter - Backwards compatibility wrapper
* Wraps existing CSV parser to implement universal DataParser interface
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.CSVParserAdapter = void 0;
exports.createCSVParserAdapter = createCSVParserAdapter;
const csv_parser_1 = require("../csv-parser");
const csv_detector_1 = require("../csv-detector");
const types_1 = require("../../core/types");
const logger_1 = require("../../utils/logger");
/**
* Adapter to make existing CSV parser compatible with universal DataParser interface
*/
class CSVParserAdapter {
csvParser;
csvDetector;
startTime;
constructor(options = {}) {
this.startTime = Date.now();
// Convert universal options to CSV-specific options
const csvOptions = {
delimiter: options.delimiter || ',',
quote: options.quote || '"',
hasHeader: options.hasHeader ?? true,
maxRows: options.maxRows,
encoding: options.encoding || 'utf8',
chunkSize: options.chunkSize || 8192,
};
this.csvParser = new csv_parser_1.CSVParser(csvOptions);
this.csvDetector = new csv_detector_1.CSVDetector();
}
/**
* Parse CSV file using existing parser
*/
async *parse(filePath, options) {
try {
// Merge options if provided
if (options) {
const mergedOptions = {
delimiter: options.delimiter || ',',
quote: options.quote || '"',
hasHeader: options.hasHeader ?? true,
maxRows: options.maxRows,
encoding: options.encoding || 'utf8',
chunkSize: options.chunkSize || 8192,
};
// Create new parser with merged options
const parser = new csv_parser_1.CSVParser(mergedOptions);
const rows = await parser.parseFile(filePath);
yield* this.adaptRowsToIterator(rows);
}
else {
// Use existing parser
const rows = await this.csvParser.parseFile(filePath);
yield* this.adaptRowsToIterator(rows);
}
}
catch (error) {
throw new types_1.DataPilotError(`CSV parsing failed: ${error.message}`, 'CSV_PARSING_ERROR', types_1.ErrorSeverity.HIGH, types_1.ErrorCategory.PARSING);
}
}
/**
* Adapt CSV parser rows to universal format iterator
*/
async *adaptRowsToIterator(rows) {
for (let index = 0; index < rows.length; index++) {
const csvRow = rows[index];
yield {
index,
data: Array.isArray(csvRow.data) ? csvRow.data : csvRow,
raw: Array.isArray(csvRow.data) ? csvRow.data.join(',') : csvRow.join(','),
metadata: {
originalType: 'csv',
...csvRow.metadata,
},
};
}
}
/**
* Detect CSV format using existing detector
*/
async detect(filePath) {
try {
// Read file sample for detection
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
const fileBuffer = await fs.promises.readFile(filePath);
const sampleBuffer = fileBuffer.slice(0, 8192); // 8KB sample
const csvDetection = csv_detector_1.CSVDetector.detect(sampleBuffer);
return {
format: 'csv',
confidence: csvDetection.delimiterConfidence || 0.9,
metadata: {
delimiter: csvDetection.delimiter,
quote: csvDetection.quote,
hasHeader: csvDetection.hasHeader,
encoding: csvDetection.encoding,
headerConfidence: csvDetection.headerConfidence,
},
encoding: csvDetection.encoding,
estimatedRows: 0, // CSV detector doesn't provide row count from sample
estimatedColumns: 0, // CSV detector doesn't provide column count from sample
suggestedOptions: {
delimiter: csvDetection.delimiter,
quote: csvDetection.quote,
hasHeader: csvDetection.hasHeader,
encoding: csvDetection.encoding,
},
};
}
catch (error) {
logger_1.logger.warn(`CSV detection failed: ${error.message}`);
return {
format: 'csv',
confidence: 0,
metadata: { error: error.message },
};
}
}
/**
* Get parsing statistics
*/
getStats() {
const csvStats = this.csvParser.getStats();
return {
bytesProcessed: csvStats.bytesProcessed || 0,
rowsProcessed: csvStats.rowsProcessed || 0,
errors: csvStats.errors?.map((err) => ({
row: err.row || 0,
column: err.column,
message: err.message || '',
code: err.code || 'CSV_ERROR',
})) || [],
startTime: this.startTime,
endTime: Date.now(),
peakMemoryUsage: csvStats.peakMemoryUsage,
format: 'csv',
};
}
/**
* Abort parsing operation
*/
abort() {
if (this.csvParser && typeof this.csvParser.abort === 'function') {
this.csvParser.abort();
}
}
/**
* Validate CSV file can be parsed
*/
async validate(filePath) {
try {
const detection = await this.detect(filePath);
const confidence = detection.confidence;
const isValid = confidence > 0.8;
const canProceed = confidence > 0.5;
const errors = [];
const warnings = [];
const suggestedFixes = [];
if (!isValid) {
errors.push('Not a valid CSV file or low confidence detection');
}
if (confidence < 0.9 && confidence > 0.8) {
warnings.push('Low confidence CSV detection - file may have parsing issues');
}
if (confidence < 0.8) {
suggestedFixes.push('Try specifying delimiter explicitly: --delimiter ","');
suggestedFixes.push('Check if file uses different quote character: --quote "\'"');
suggestedFixes.push('Verify file encoding is correct');
}
// Check for specific CSV issues
if (detection.metadata.delimiter === 'unknown') {
warnings.push('Could not reliably detect delimiter');
suggestedFixes.push('Specify delimiter manually: --delimiter ";"');
}
return {
valid: isValid,
errors,
warnings,
canProceed,
suggestedFixes,
};
}
catch (error) {
return {
valid: false,
errors: [error.message],
warnings: [],
canProceed: false,
suggestedFixes: ['Check file exists and is readable', 'Verify file format is CSV'],
};
}
}
/**
* Get supported file extensions
*/
getSupportedExtensions() {
return ['.csv'];
}
/**
* Get format name
*/
getFormatName() {
return 'csv';
}
}
exports.CSVParserAdapter = CSVParserAdapter;
/**
* Factory function to create CSV parser adapter
*/
function createCSVParserAdapter(options) {
return new CSVParserAdapter(options);
}
//# sourceMappingURL=csv-parser-adapter.js.map