datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
365 lines • 14.9 kB
JavaScript
"use strict";
/**
* Section 2: Uniqueness Dimension Analyzer
* Detects exact and semantic duplicates, analyzes cardinality
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.UniquenessAnalyzer = void 0;
const types_1 = require("../../core/types");
const logger_1 = require("../../utils/logger");
class UniquenessAnalyzer {
data;
headers;
columnTypes;
rowCount;
potentialKeys;
constructor(input) {
this.data = input.data;
this.headers = input.headers;
this.columnTypes = input.columnTypes;
this.rowCount = input.rowCount;
this.potentialKeys = input.potentialKeys || this.inferPotentialKeys();
}
analyze() {
const start = performance.now();
// 1. Exact duplicate detection
const exactDuplicates = this.detectExactDuplicates();
// 2. Key uniqueness analysis
const keyUniqueness = this.analyzeKeyUniqueness();
// 3. Column-level uniqueness
const columnUniqueness = this.analyzeColumnUniqueness();
// 4. Semantic duplicate detection (limited scope for performance)
const semanticDuplicates = this.detectSemanticDuplicates();
// 5. Calculate overall score
const score = this.calculateUniquenessScore(exactDuplicates, keyUniqueness, columnUniqueness);
logger_1.logger.debug(`Uniqueness analysis completed in ${(performance.now() - start).toFixed(2)}ms`, { operation: 'uniqueness-analysis' });
return {
exactDuplicates,
keyUniqueness,
columnUniqueness,
semanticDuplicates,
score,
};
}
detectExactDuplicates() {
const rowHashes = new Map();
// Create hash for each row and track indices
for (let rowIdx = 0; rowIdx < this.rowCount; rowIdx++) {
const row = this.data[rowIdx] || [];
const rowHash = this.createRowHash(row);
if (!rowHashes.has(rowHash)) {
rowHashes.set(rowHash, []);
}
rowHashes.get(rowHash).push(rowIdx);
}
// Find duplicates
const duplicateGroups = [];
let totalDuplicateRows = 0;
for (const [, indices] of rowHashes.entries()) {
if (indices.length > 1) {
duplicateGroups.push({
rowIndices: indices,
duplicateType: 'Exact',
});
totalDuplicateRows += indices.length - 1; // Don't count the original
}
}
const percentage = (totalDuplicateRows / this.rowCount) * 100;
return {
count: totalDuplicateRows,
percentage,
duplicateGroups: duplicateGroups.slice(0, 10), // Limit for performance
};
}
analyzeKeyUniqueness() {
const keyColumns = this.potentialKeys
.map((keyName) => {
const colIdx = this.headers.indexOf(keyName);
return colIdx !== -1 ? colIdx : null;
})
.filter((idx) => idx !== null);
// Also analyze columns that look like IDs
const idLikeColumns = this.headers
.map((header, idx) => ({ header: header.toLowerCase(), idx }))
.filter(({ header }) => header.includes('id') ||
header.includes('key') ||
header.includes('code') ||
header === 'index')
.map(({ idx }) => idx);
const allCandidateColumns = [...new Set([...keyColumns, ...idLikeColumns])];
return allCandidateColumns.map((colIdx) => {
const columnName = this.headers[colIdx];
const valueMap = new Map();
let nonNullCount = 0;
// Count occurrences of each value
for (let rowIdx = 0; rowIdx < this.rowCount; rowIdx++) {
const value = this.normalizeValue(this.data[rowIdx]?.[colIdx]);
if (value !== null) {
nonNullCount++;
const key = String(value);
valueMap.set(key, (valueMap.get(key) || 0) + 1);
}
}
const uniqueCount = valueMap.size;
const duplicateCount = nonNullCount - uniqueCount;
const cardinality = uniqueCount;
// Find most frequent duplicates
const duplicateValues = Array.from(valueMap.entries())
.filter(([_, count]) => count > 1)
.map(([value, frequency]) => ({ value, frequency }))
.sort((a, b) => b.frequency - a.frequency)
.slice(0, 5);
const isPrimaryKey = this.isPotentialPrimaryKey(columnName, duplicateCount, nonNullCount);
return {
columnName,
isPrimaryKey,
duplicateCount,
cardinality,
duplicateValues,
};
});
}
analyzeColumnUniqueness() {
return this.headers.map((columnName, colIdx) => {
const valueMap = new Map();
let nonNullCount = 0;
for (let rowIdx = 0; rowIdx < this.rowCount; rowIdx++) {
const value = this.normalizeValue(this.data[rowIdx]?.[colIdx]);
if (value !== null) {
nonNullCount++;
const key = String(value);
valueMap.set(key, (valueMap.get(key) || 0) + 1);
}
}
const uniqueCount = valueMap.size;
const uniquePercentage = nonNullCount > 0 ? (uniqueCount / nonNullCount) * 100 : 0;
const duplicateCount = nonNullCount - uniqueCount;
// Find most frequent duplicate
let mostFrequentDuplicate = undefined;
let maxFrequency = 1;
for (const [value, frequency] of valueMap.entries()) {
if (frequency > maxFrequency) {
maxFrequency = frequency;
mostFrequentDuplicate = { value, frequency };
}
}
return {
columnName,
uniquePercentage,
duplicateCount,
mostFrequentDuplicate,
};
});
}
detectSemanticDuplicates() {
const semanticDuplicates = [];
const methods = ['levenshtein', 'soundex'];
// For performance, only check string columns with reasonable cardinality
const stringColumns = this.headers
.map((header, idx) => ({ header, idx, type: this.columnTypes[idx] }))
.filter(({ type }) => type === types_1.DataType.STRING)
.slice(0, 3); // Limit to first 3 string columns for performance
if (stringColumns.length === 0) {
return {
suspectedPairs: 0,
duplicates: [],
methods: [],
};
}
// Sample rows for performance (max 1000 comparisons)
const maxComparisons = 1000;
const sampleSize = Math.min(this.rowCount, Math.floor(Math.sqrt(maxComparisons * 2)));
const sampleIndices = this.getSampleIndices(sampleSize);
for (let i = 0; i < sampleIndices.length; i++) {
for (let j = i + 1; j < sampleIndices.length; j++) {
const rowIdx1 = sampleIndices[i];
const rowIdx2 = sampleIndices[j];
const similarity = this.calculateRowSimilarity(rowIdx1, rowIdx2, stringColumns);
if (similarity.overall > 0.8) {
// High similarity threshold
semanticDuplicates.push({
recordPair: [rowIdx1, rowIdx2],
confidence: similarity.overall,
method: 'composite',
similarity: similarity.columnSimilarities,
});
}
}
}
return {
suspectedPairs: semanticDuplicates.length,
duplicates: semanticDuplicates.slice(0, 10), // Limit results
methods,
};
}
calculateRowSimilarity(rowIdx1, rowIdx2, stringColumns) {
const columnSimilarities = {};
let totalSimilarity = 0;
let validColumns = 0;
for (const { header, idx } of stringColumns) {
const value1 = this.normalizeValue(this.data[rowIdx1]?.[idx]);
const value2 = this.normalizeValue(this.data[rowIdx2]?.[idx]);
if (value1 !== null && value2 !== null) {
const str1 = String(value1).toLowerCase().trim();
const str2 = String(value2).toLowerCase().trim();
if (str1.length > 0 && str2.length > 0) {
const levenshteinSim = this.calculateLevenshteinSimilarity(str1, str2);
const soundexSim = this.calculateSoundexSimilarity(str1, str2);
const columnSim = Math.max(levenshteinSim, soundexSim);
columnSimilarities[header] = columnSim;
totalSimilarity += columnSim;
validColumns++;
}
}
}
const overall = validColumns > 0 ? totalSimilarity / validColumns : 0;
return {
overall,
columnSimilarities,
};
}
calculateLevenshteinSimilarity(str1, str2) {
const distance = this.levenshteinDistance(str1, str2);
const maxLength = Math.max(str1.length, str2.length);
return maxLength === 0 ? 1 : 1 - distance / maxLength;
}
levenshteinDistance(str1, str2) {
const matrix = Array(str2.length + 1)
.fill(null)
.map(() => Array(str1.length + 1).fill(null));
for (let i = 0; i <= str1.length; i++)
matrix[0][i] = i;
for (let j = 0; j <= str2.length; j++)
matrix[j][0] = j;
for (let j = 1; j <= str2.length; j++) {
for (let i = 1; i <= str1.length; i++) {
const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
matrix[j][i] = Math.min(matrix[j][i - 1] + 1, // deletion
matrix[j - 1][i] + 1, // insertion
matrix[j - 1][i - 1] + indicator);
}
}
return matrix[str2.length][str1.length];
}
calculateSoundexSimilarity(str1, str2) {
const soundex1 = this.soundex(str1);
const soundex2 = this.soundex(str2);
return soundex1 === soundex2 ? 1 : 0;
}
soundex(str) {
// Simplified Soundex implementation
const mapping = {
b: '1',
f: '1',
p: '1',
v: '1',
c: '2',
g: '2',
j: '2',
k: '2',
q: '2',
s: '2',
x: '2',
z: '2',
d: '3',
t: '3',
l: '4',
m: '5',
n: '5',
r: '6',
};
const cleaned = str.toLowerCase().replace(/[^a-z]/g, '');
if (cleaned.length === 0)
return '0000';
let result = cleaned[0].toUpperCase();
let prev = mapping[cleaned[0]] || '0';
for (let i = 1; i < cleaned.length && result.length < 4; i++) {
const code = mapping[cleaned[i]] || '0';
if (code !== '0' && code !== prev) {
result += code;
}
prev = code;
}
return result.padEnd(4, '0').substring(0, 4);
}
getSampleIndices(sampleSize) {
if (sampleSize >= this.rowCount) {
return Array.from({ length: this.rowCount }, (_, i) => i);
}
const indices = [];
const step = this.rowCount / sampleSize;
for (let i = 0; i < sampleSize; i++) {
indices.push(Math.floor(i * step));
}
return indices;
}
calculateUniquenessScore(exactDuplicates, keyUniqueness, columnUniqueness) {
// Base score starts at 100
let score = 100;
// Penalize for exact duplicates
score -= Math.min(30, exactDuplicates.percentage * 2); // Max 30 points off
// Penalize for key constraint violations
const keyViolations = keyUniqueness.filter((key) => key.isPrimaryKey && key.duplicateCount > 0);
score -= keyViolations.length * 15; // 15 points per violated key
// Slight penalty for low uniqueness in key-like columns
const lowUniquenessColumns = columnUniqueness.filter((col) => {
const isKeyLike = col.columnName.toLowerCase().includes('id') || col.columnName.toLowerCase().includes('key');
return isKeyLike && col.uniquePercentage < 80;
});
score -= lowUniquenessColumns.length * 5; // 5 points per low-uniqueness key column
score = Math.max(0, score);
let interpretation;
if (score >= 95)
interpretation = 'Excellent';
else if (score >= 85)
interpretation = 'Good';
else if (score >= 70)
interpretation = 'Fair';
else if (score >= 50)
interpretation = 'Needs Improvement';
else
interpretation = 'Poor';
return {
score: Math.round(score * 100) / 100,
interpretation,
details: `${exactDuplicates.percentage.toFixed(2)}% duplicate rows, ${keyViolations.length} key constraint violations`,
};
}
createRowHash(row) {
return row.map((cell) => String(this.normalizeValue(cell) || '')).join('|');
}
normalizeValue(value) {
if (value === null || value === undefined)
return null;
const trimmed = String(value).trim();
if (trimmed === '' || trimmed.toLowerCase() === 'null' || trimmed.toLowerCase() === 'na') {
return null;
}
return trimmed;
}
inferPotentialKeys() {
const keys = [];
for (const header of this.headers) {
const lower = header.toLowerCase();
if (lower.includes('id') ||
lower.includes('key') ||
lower.includes('code') ||
lower === 'index' ||
lower.endsWith('_id') ||
lower.endsWith('id') ||
lower.startsWith('id_')) {
keys.push(header);
}
}
return keys;
}
isPotentialPrimaryKey(columnName, duplicateCount, totalCount) {
const lower = columnName.toLowerCase();
const isKeyLike = lower.includes('id') || lower.includes('key') || lower === 'index';
const hasNoDuplicates = duplicateCount === 0;
const highCardinality = totalCount > 0 && (totalCount - duplicateCount) / totalCount > 0.9;
return isKeyLike && hasNoDuplicates && highCardinality;
}
}
exports.UniquenessAnalyzer = UniquenessAnalyzer;
//# sourceMappingURL=uniqueness-analyzer.js.map