datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
475 lines • 19.2 kB
JavaScript
;
/**
* File Metadata Collector - Advanced file system analysis
* Handles file stats, hashing, MIME detection with privacy controls
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.FileMetadataCollector = void 0;
const crypto_1 = require("crypto");
const fs_1 = require("fs");
const path_1 = require("path");
const zlib_1 = require("zlib");
class FileMetadataCollector {
config;
warnings = [];
constructor(config) {
this.config = config;
}
/**
* Collect comprehensive file metadata with optional hashing
*/
async collectMetadata(filePath) {
try {
// Resolve absolute path
const fullPath = (0, path_1.resolve)(filePath);
const filename = (0, path_1.basename)(fullPath);
// Get file statistics
const stats = (0, fs_1.statSync)(fullPath);
// Calculate file size
const fileSizeBytes = stats.size;
const fileSizeMB = fileSizeBytes / (1024 * 1024);
// Detect MIME type
const mimeType = this.detectMimeType(fullPath);
// Calculate hash if enabled
const sha256Hash = this.config.enableFileHashing
? await this.calculateFileHash(fullPath)
: 'disabled';
// Apply privacy controls to path
const sanitizedPath = this.sanitizePath(fullPath);
const metadata = {
originalFilename: filename,
fullResolvedPath: sanitizedPath,
fileSizeBytes,
fileSizeMB: Number(fileSizeMB.toFixed(6)),
mimeType,
lastModified: stats.mtime,
sha256Hash,
};
// Add compression analysis if enabled
if (this.config.enableCompressionAnalysis) {
try {
metadata.compressionAnalysis = await this.analyzeCompression(fullPath, fileSizeBytes);
}
catch (error) {
// Fall back to basic compression info if analysis fails
metadata.compressionAnalysis = {
originalSizeBytes: fileSizeBytes,
estimatedGzipSizeBytes: Math.round(fileSizeBytes * 0.7),
estimatedGzipReduction: 30,
estimatedParquetSizeBytes: Math.round(fileSizeBytes * 0.5),
estimatedParquetReduction: 50,
columnEntropy: [],
recommendedFormat: 'gzip',
analysisMethod: 'Fallback estimation (analysis failed)',
};
}
}
// Add health check if enabled
if (this.config.enableHealthChecks) {
try {
metadata.healthCheck = await this.performHealthCheck(fullPath);
}
catch (error) {
// Fall back to basic health info if check fails
metadata.healthCheck = {
bomDetected: false,
lineEndingConsistency: 'unknown',
nullBytesDetected: false,
validEncodingThroughout: true,
largeFileWarning: fileSizeMB > 100,
recommendations: ['Health check failed - using fallback assessment'],
healthScore: 75,
};
}
}
// Performance info for very large files
if (fileSizeMB > 1000) {
// Only warn for files > 1GB
this.warnings.push({
category: 'file',
severity: 'low',
message: `Very large file detected (${fileSizeMB.toFixed(1)}MB)`,
impact: 'Analysis will use streaming algorithms to manage memory efficiently',
suggestion: 'Processing may take longer but memory usage will remain bounded',
});
}
return metadata;
}
catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
throw new Error(`Failed to collect file metadata: ${message}`);
}
}
/**
* Detect MIME type with fallback strategies
*/
detectMimeType(filePath) {
const ext = (0, path_1.extname)(filePath).toLowerCase();
// Common MIME types for data files
const mimeTypes = {
'.csv': 'text/csv',
'.tsv': 'text/tab-separated-values',
'.txt': 'text/plain',
'.json': 'application/json',
'.xml': 'application/xml',
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'.xls': 'application/vnd.ms-excel',
};
return mimeTypes[ext] || 'application/octet-stream';
}
/**
* Calculate SHA256 hash efficiently for large files
*/
async calculateFileHash(filePath) {
return new Promise((resolve, reject) => {
const hash = (0, crypto_1.createHash)('sha256');
const stream = (0, fs_1.createReadStream)(filePath, { highWaterMark: 64 * 1024 });
stream.on('data', (chunk) => {
hash.update(chunk);
});
stream.on('end', () => {
resolve(hash.digest('hex'));
});
stream.on('error', (error) => {
reject(new Error(`Hash calculation failed: ${error.message}`));
});
});
}
/**
* Apply privacy controls to file paths
*/
sanitizePath(fullPath) {
switch (this.config.privacyMode) {
case 'minimal':
return (0, path_1.basename)(fullPath);
case 'redacted':
// Replace user directory with placeholder
return fullPath
.replace(/\/Users\/[^\/]+/, '/Users/[user]')
.replace(/\/home\/[^\/]+/, '/home/[user]')
.replace(/C:\\Users\\[^\\]+/, 'C:\\Users\\[user]')
// GitHub Actions Windows CI path: C:\a\repo\repo\...
.replace(/C:\\a\\[^\\]+\\[^\\]+/, 'C:\\a\\[project]\\[project]')
// Azure DevOps Windows CI paths
.replace(/C:\\Agent\\_work\\[^\\]+/, 'C:\\Agent\\_work\\[build]')
.replace(/D:\\a\\[^\\]+/, 'D:\\a\\[build]');
case 'full':
default:
return fullPath;
}
}
/**
* Analyze compression potential and estimate savings
*/
async analyzeCompression(filePath, originalSize) {
try {
// Sample the file for compression analysis (first 1MB or full file if smaller)
const sampleSize = Math.min(originalSize, 1024 * 1024);
const sampleData = await this.readFileSample(filePath, sampleSize);
// Calculate gzip compression
const gzippedSample = (0, zlib_1.gzipSync)(sampleData);
const sampleCompressionRatio = gzippedSample.length / sampleData.length;
const estimatedGzipSize = Math.round(originalSize * sampleCompressionRatio);
const gzipReduction = Math.round((1 - sampleCompressionRatio) * 100);
// Estimate Parquet compression (roughly 40-60% of original for text data)
const estimatedParquetRatio = 0.5; // Conservative estimate for CSV->Parquet
const estimatedParquetSize = Math.round(originalSize * estimatedParquetRatio);
const parquetReduction = Math.round((1 - estimatedParquetRatio) * 100);
// Calculate Shannon entropy for sample columns (simplified)
const columnEntropy = this.calculateColumnEntropy(sampleData.toString('utf8'));
// Determine recommendation
let recommendedFormat = 'none';
if (gzipReduction > 30) {
recommendedFormat = gzipReduction > parquetReduction ? 'gzip' : 'parquet';
}
return {
originalSizeBytes: originalSize,
estimatedGzipSizeBytes: estimatedGzipSize,
estimatedGzipReduction: gzipReduction,
estimatedParquetSizeBytes: estimatedParquetSize,
estimatedParquetReduction: parquetReduction,
columnEntropy,
recommendedFormat,
analysisMethod: `Sample-based analysis (${(sampleSize / 1024).toFixed(0)}KB sample)`,
};
}
catch (error) {
// Fallback analysis if compression fails
return {
originalSizeBytes: originalSize,
estimatedGzipSizeBytes: Math.round(originalSize * 0.7),
estimatedGzipReduction: 30,
estimatedParquetSizeBytes: Math.round(originalSize * 0.5),
estimatedParquetReduction: 50,
columnEntropy: [],
recommendedFormat: 'gzip',
analysisMethod: 'Fallback estimation (compression test failed)',
};
}
}
/**
* Perform file health checks
*/
async performHealthCheck(filePath) {
try {
// Read first part of file for analysis
const sampleSize = Math.min(64 * 1024, (0, fs_1.statSync)(filePath).size); // 64KB sample
const sampleData = await this.readFileSample(filePath, sampleSize);
const sampleText = sampleData.toString('utf8');
// Check for BOM
const bomDetected = sampleData.length >= 3 &&
sampleData[0] === 0xEF &&
sampleData[1] === 0xBB &&
sampleData[2] === 0xBF;
// Check line ending consistency
const crlfCount = (sampleText.match(/\r\n/g) || []).length;
const lfCount = (sampleText.match(/(?<!\r)\n/g) || []).length;
const crCount = (sampleText.match(/\r(?!\n)/g) || []).length;
let lineEndingConsistency = 'unknown';
const totalLineEndings = crlfCount + lfCount + crCount;
if (totalLineEndings > 0) {
const dominant = Math.max(crlfCount, lfCount, crCount);
lineEndingConsistency = (dominant / totalLineEndings) > 0.9 ? 'consistent' : 'mixed';
}
// Check for null bytes
const nullBytesDetected = sampleData.includes(0);
// Check encoding validity
let validEncodingThroughout = true;
try {
sampleData.toString('utf8');
}
catch {
validEncodingThroughout = false;
}
// Large file warning
const fileSizeMB = (0, fs_1.statSync)(filePath).size / (1024 * 1024);
const largeFileWarning = fileSizeMB > 100; // 100MB threshold
// Generate recommendations
const recommendations = [];
if (bomDetected) {
recommendations.push('Consider removing BOM for better compatibility');
}
if (lineEndingConsistency === 'mixed') {
recommendations.push('Standardise line endings for consistent processing');
}
if (nullBytesDetected) {
recommendations.push('File contains null bytes - verify data integrity');
}
if (!validEncodingThroughout) {
recommendations.push('File contains invalid UTF-8 sequences');
}
if (largeFileWarning) {
recommendations.push('Large file detected - consider chunked processing');
}
// Calculate health score
let healthScore = 100;
if (bomDetected)
healthScore -= 5;
if (lineEndingConsistency === 'mixed')
healthScore -= 10;
if (nullBytesDetected)
healthScore -= 20;
if (!validEncodingThroughout)
healthScore -= 30;
if (largeFileWarning)
healthScore -= 5;
return {
bomDetected,
bomType: bomDetected ? 'UTF-8' : undefined,
lineEndingConsistency,
nullBytesDetected,
validEncodingThroughout,
largeFileWarning,
recommendations,
healthScore: Math.max(0, healthScore),
};
}
catch (error) {
return {
bomDetected: false,
lineEndingConsistency: 'unknown',
nullBytesDetected: false,
validEncodingThroughout: true,
largeFileWarning: false,
recommendations: ['Health check failed - file may be inaccessible'],
healthScore: 50,
};
}
}
/**
* Calculate column entropy for compression analysis
*/
calculateColumnEntropy(sampleText) {
try {
const lines = sampleText.split('\n').slice(0, 100); // First 100 lines
if (lines.length < 2)
return [];
const headers = lines[0].split(',').map(h => h.trim().replace(/"/g, ''));
const columnEntropy = [];
// Analyse first few columns (limit to 10 for performance)
const maxColumns = Math.min(headers.length, 10);
for (let i = 0; i < maxColumns; i++) {
const values = [];
for (let j = 1; j < lines.length && j < 50; j++) { // Sample 50 rows
const columns = lines[j].split(',');
if (columns[i]) {
values.push(columns[i].trim().replace(/"/g, ''));
}
}
if (values.length > 0) {
const entropy = this.calculateShannonEntropy(values);
let compressionPotential = 'medium';
if (entropy < 2)
compressionPotential = 'high';
else if (entropy > 4)
compressionPotential = 'low';
columnEntropy.push({
columnName: headers[i] || `Column_${i + 1}`,
entropy: Math.round(entropy * 100) / 100,
compressionPotential,
});
}
}
return columnEntropy;
}
catch {
return [];
}
}
/**
* Calculate Shannon entropy for a set of values
*/
calculateShannonEntropy(values) {
const frequency = new Map();
for (const value of values) {
frequency.set(value, (frequency.get(value) || 0) + 1);
}
const total = values.length;
let entropy = 0;
for (const count of frequency.values()) {
const probability = count / total;
entropy -= probability * Math.log2(probability);
}
return entropy;
}
/**
* Read a sample of the file
*/
async readFileSample(filePath, sampleSize) {
return new Promise((resolve, reject) => {
const chunks = [];
let totalRead = 0;
let resolved = false;
const stream = (0, fs_1.createReadStream)(filePath, { start: 0, end: sampleSize - 1 });
const resolveOnce = (result) => {
if (!resolved) {
resolved = true;
resolve(result);
}
};
const rejectOnce = (error) => {
if (!resolved) {
resolved = true;
reject(error);
}
};
stream.on('data', (chunk) => {
chunks.push(chunk);
totalRead += chunk.length;
if (totalRead >= sampleSize) {
stream.destroy();
resolveOnce(Buffer.concat(chunks));
}
});
stream.on('end', () => {
resolveOnce(Buffer.concat(chunks));
});
stream.on('error', (error) => {
rejectOnce(error);
});
stream.on('close', () => {
// Ensure we resolve even if end event doesn't fire after destroy
resolveOnce(Buffer.concat(chunks));
});
});
}
/**
* Get collected warnings
*/
getWarnings() {
return [...this.warnings];
}
/**
* Clear warnings
*/
clearWarnings() {
this.warnings = [];
}
/**
* Validate file accessibility and basic requirements
*/
validateFile(filePath) {
const errors = [];
// Check if file exists first
if (!(0, fs_1.existsSync)(filePath)) {
errors.push(`File does not exist: ${filePath}`);
return { valid: false, errors };
}
try {
// Check file accessibility
(0, fs_1.accessSync)(filePath, fs_1.constants.R_OK);
}
catch (error) {
if (error instanceof Error) {
const nodeError = error;
if (nodeError.code === 'EACCES') {
errors.push(`File is not readable - check permissions: ${filePath}`);
}
else {
errors.push(`File access error: ${nodeError.message}`);
}
}
else {
errors.push('File access error: Unknown error');
}
return { valid: false, errors };
}
try {
const stats = (0, fs_1.statSync)(filePath);
if (!stats.isFile()) {
errors.push(`Path does not point to a regular file: ${filePath}`);
}
else {
// Only check file-specific properties if it's actually a file
if (stats.size === 0) {
errors.push(`File is empty: ${filePath}`);
}
if (stats.size > 10 * 1024 * 1024 * 1024) {
// 10GB
errors.push(`File exceeds maximum size limit (10GB): ${filePath}`);
}
}
}
catch (error) {
// Handle stat errors
if (error instanceof Error) {
const nodeError = error;
if (nodeError.code === 'EISDIR') {
errors.push(`Path points to a directory, not a file: ${filePath}`);
}
else {
errors.push(`File stat error: ${nodeError.message}`);
}
}
else {
errors.push('File stat error: Unknown error');
}
}
return {
valid: errors.length === 0,
errors,
};
}
}
exports.FileMetadataCollector = FileMetadataCollector;
//# sourceMappingURL=file-metadata-collector.js.map