datapilot-cli
Version:
Enterprise-grade streaming multi-format data analysis with comprehensive statistical insights and intelligent relationship detection - supports CSV, JSON, Excel, TSV, Parquet - memory-efficient, cross-platform
515 lines • 19 kB
JavaScript
"use strict";
/**
* Statistical Tests Library
* Implements proper statistical tests for EDA analysis
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.CorrelationSignificanceTest = exports.ChiSquaredTest = exports.ANOVATest = exports.KolmogorovSmirnovTest = exports.JarqueBeraTest = exports.ShapiroWilkTest = void 0;
/**
* Shapiro-Wilk Test for Normality
* Tests the null hypothesis that data comes from a normal distribution
*/
class ShapiroWilkTest {
static COEFFICIENTS = [
// Coefficients for Shapiro-Wilk test (approximation for small samples)
// Full implementation would require extensive coefficient tables
[0.7071, 0.0, 0.0, 0.0, 0.0],
[0.7071, 0.7071, 0.0, 0.0, 0.0],
[0.6872, 0.1677, 0.6872, 0.0, 0.0],
[0.6646, 0.2413, 0.2413, 0.6646, 0.0],
[0.6431, 0.2806, 0.0875, 0.2806, 0.6431],
];
/**
* Standard normal cumulative distribution function
*/
static standardNormalCDF(z) {
// Approximation using error function
const t = 1.0 / (1.0 + 0.2316419 * Math.abs(z));
const d = 0.3989423 * Math.exp((-z * z) / 2);
const prob = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))));
return z >= 0 ? 1 - prob : prob;
}
static test(data) {
if (data.length < 3) {
return {
statistic: 0,
pValue: 1,
interpretation: 'Insufficient data for Shapiro-Wilk test (n < 3)',
};
}
if (data.length > 5000) {
return {
statistic: 0,
pValue: 1,
interpretation: 'Sample too large for Shapiro-Wilk test (use Kolmogorov-Smirnov instead)',
};
}
// Sort the data
const sortedData = [...data].sort((a, b) => a - b);
const n = sortedData.length;
// Calculate sample mean and variance
const mean = sortedData.reduce((sum, val) => sum + val, 0) / n;
const variance = sortedData.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / (n - 1);
if (variance === 0) {
return {
statistic: 1,
pValue: 1,
interpretation: 'All values identical - cannot test normality',
};
}
// Simplified Shapiro-Wilk calculation
// Full implementation requires extensive coefficient tables and complex calculations
let b = 0;
const m = Math.floor(n / 2);
for (let i = 0; i < m; i++) {
const coeff = i < 5 && n <= 10 ? this.COEFFICIENTS[Math.min(n - 3, 4)][i] : 0.5;
b += coeff * (sortedData[n - 1 - i] - sortedData[i]);
}
const w = (b * b) / ((n - 1) * variance);
// Improved p-value calculation using statistical approximation
// Based on Royston's approximation for Shapiro-Wilk p-values
let pValue;
// Log transformation for better p-value estimation
const logW = Math.log(1 - w);
// Approximation based on sample size and W statistic
if (n <= 11) {
// For small samples, use conservative estimate
if (w > 0.987) {
pValue = 1 - Math.exp(-0.5 * Math.pow((w - 0.987) / 0.013, 2));
}
else if (w > 0.95) {
pValue = 0.2 + (0.3 * (w - 0.95)) / 0.037;
}
else if (w > 0.9) {
pValue = 0.05 + (0.15 * (w - 0.9)) / 0.05;
}
else {
pValue = 0.01 + 0.04 * Math.max(0, (w - 0.8) / 0.1);
}
}
else {
// For larger samples, use asymptotic approximation
const mu = -1.273 + 2.706 * Math.pow(n, -0.5);
const sigma = 1.038 + 0.15 * Math.pow(n, -0.8);
const z = (logW - mu) / sigma;
// Standard normal approximation
pValue = 1 - this.standardNormalCDF(z);
}
// Ensure p-value is in valid range
pValue = Math.max(0.001, Math.min(0.999, pValue));
const interpretation = pValue > 0.05
? 'Data consistent with normal distribution (p > 0.05)'
: 'Data significantly deviates from normal distribution (p ≤ 0.05)';
return {
statistic: Number(w.toFixed(6)),
pValue: Number(pValue.toFixed(4)),
interpretation,
};
}
}
exports.ShapiroWilkTest = ShapiroWilkTest;
/**
* Jarque-Bera Test for Normality
* Tests normality using skewness and kurtosis
*/
class JarqueBeraTest {
static test(data) {
if (data.length < 3) {
return {
statistic: 0,
pValue: 1,
interpretation: 'Insufficient data for Jarque-Bera test (n < 3)',
};
}
const n = data.length;
const mean = data.reduce((sum, val) => sum + val, 0) / n;
// Calculate moments
let m2 = 0, m3 = 0, m4 = 0;
for (const value of data) {
const deviation = value - mean;
const deviation2 = deviation * deviation;
const deviation3 = deviation2 * deviation;
const deviation4 = deviation2 * deviation2;
m2 += deviation2;
m3 += deviation3;
m4 += deviation4;
}
m2 /= n;
m3 /= n;
m4 /= n;
if (m2 === 0) {
return {
statistic: 0,
pValue: 1,
interpretation: 'All values identical - cannot test normality',
};
}
// Calculate skewness and kurtosis
const skewness = m3 / Math.pow(m2, 1.5);
const kurtosis = m4 / (m2 * m2) - 3; // Excess kurtosis
// Jarque-Bera statistic
const jb = (n / 6) * (skewness * skewness + (kurtosis * kurtosis) / 4);
// Approximate p-value using chi-squared distribution with 2 df
// Critical values: 5.99 (p=0.05), 9.21 (p=0.01), 13.82 (p=0.001)
let pValue;
if (jb < 5.99) {
pValue = 0.2;
}
else if (jb < 9.21) {
pValue = 0.05;
}
else if (jb < 13.82) {
pValue = 0.01;
}
else {
pValue = 0.001;
}
const interpretation = pValue > 0.05
? 'Data consistent with normal distribution (p > 0.05)'
: 'Data significantly deviates from normal distribution (p ≤ 0.05)';
return {
statistic: Number(jb.toFixed(6)),
pValue: Number(pValue.toFixed(4)),
interpretation,
};
}
}
exports.JarqueBeraTest = JarqueBeraTest;
/**
* Kolmogorov-Smirnov Test for Normality
* Tests if data follows a normal distribution
*/
class KolmogorovSmirnovTest {
static test(data) {
if (data.length < 5) {
return {
statistic: 0,
pValue: 1,
interpretation: 'Insufficient data for Kolmogorov-Smirnov test (n < 5)',
};
}
const n = data.length;
const sortedData = [...data].sort((a, b) => a - b);
// Calculate sample mean and standard deviation
const mean = sortedData.reduce((sum, val) => sum + val, 0) / n;
const variance = sortedData.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / (n - 1);
const stdDev = Math.sqrt(variance);
if (stdDev === 0) {
return {
statistic: 0,
pValue: 1,
interpretation: 'All values identical - cannot test normality',
};
}
// Calculate D statistic (maximum difference between empirical and theoretical CDF)
let maxDifference = 0;
for (let i = 0; i < n; i++) {
const empiricalCDF = (i + 1) / n;
const standardized = (sortedData[i] - mean) / stdDev;
const theoreticalCDF = this.normalCDF(standardized);
const difference = Math.abs(empiricalCDF - theoreticalCDF);
maxDifference = Math.max(maxDifference, difference);
}
// Approximate critical values for K-S test
const criticalValue005 = 1.36 / Math.sqrt(n);
const criticalValue001 = 1.63 / Math.sqrt(n);
let pValue;
if (maxDifference < criticalValue005) {
pValue = 0.2;
}
else if (maxDifference < criticalValue001) {
pValue = 0.05;
}
else {
pValue = 0.01;
}
const interpretation = pValue > 0.05
? 'Data consistent with normal distribution (p > 0.05)'
: 'Data significantly deviates from normal distribution (p ≤ 0.05)';
return {
statistic: Number(maxDifference.toFixed(6)),
pValue: Number(pValue.toFixed(4)),
interpretation,
};
}
/**
* Approximate normal CDF using erf approximation
*/
static normalCDF(x) {
return 0.5 * (1 + this.erf(x / Math.sqrt(2)));
}
/**
* Error function approximation
*/
static erf(x) {
// Abramowitz and Stegun approximation
const a1 = 0.254829592;
const a2 = -0.284496736;
const a3 = 1.421413741;
const a4 = -1.453152027;
const a5 = 1.061405429;
const p = 0.3275911;
const sign = x >= 0 ? 1 : -1;
x = Math.abs(x);
const t = 1.0 / (1.0 + p * x);
const y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
return sign * y;
}
}
exports.KolmogorovSmirnovTest = KolmogorovSmirnovTest;
/**
* ANOVA (Analysis of Variance) Test
* Tests if means of multiple groups are significantly different
*/
class ANOVATest {
static test(groups) {
if (groups.length < 2) {
return {
fStatistic: 0,
pValue: 1,
interpretation: 'Need at least 2 groups for ANOVA',
};
}
// Filter out empty groups
const validGroups = groups.filter((group) => group.length > 0);
if (validGroups.length < 2) {
return {
fStatistic: 0,
pValue: 1,
interpretation: 'Need at least 2 non-empty groups for ANOVA',
};
}
const k = validGroups.length; // number of groups
const n = validGroups.reduce((sum, group) => sum + group.length, 0); // total sample size
if (n <= k) {
return {
fStatistic: 0,
pValue: 1,
interpretation: 'Insufficient data for ANOVA (total n ≤ number of groups)',
};
}
// Calculate group means and overall mean
const groupMeans = validGroups.map((group) => group.reduce((sum, val) => sum + val, 0) / group.length);
const overallMean = validGroups.flat().reduce((sum, val) => sum + val, 0) / n;
// Calculate sum of squares between groups (SSB)
let ssb = 0;
for (let i = 0; i < validGroups.length; i++) {
const groupSize = validGroups[i].length;
const groupMean = groupMeans[i];
ssb += groupSize * Math.pow(groupMean - overallMean, 2);
}
// Calculate sum of squares within groups (SSW)
let ssw = 0;
for (let i = 0; i < validGroups.length; i++) {
const groupMean = groupMeans[i];
for (const value of validGroups[i]) {
ssw += Math.pow(value - groupMean, 2);
}
}
// Calculate degrees of freedom
const dfBetween = k - 1;
const dfWithin = n - k;
// Calculate mean squares
const msBetween = ssb / dfBetween;
const msWithin = ssw / dfWithin;
// Calculate F-statistic
const fStatistic = msWithin > 0 ? msBetween / msWithin : 0;
// Approximate p-value using F-distribution
// This is a simplified approximation
let pValue;
if (fStatistic < 1) {
pValue = 0.5;
}
else if (fStatistic < 2.5) {
pValue = 0.1;
}
else if (fStatistic < 4) {
pValue = 0.05;
}
else if (fStatistic < 7) {
pValue = 0.01;
}
else {
pValue = 0.001;
}
const interpretation = pValue > 0.05
? 'No significant difference between group means (p > 0.05)'
: 'Significant difference between group means (p ≤ 0.05)';
return {
fStatistic: Number(fStatistic.toFixed(6)),
pValue: Number(pValue.toFixed(4)),
interpretation,
};
}
}
exports.ANOVATest = ANOVATest;
/**
* Chi-Squared Test of Independence
* Tests association between two categorical variables
*/
class ChiSquaredTest {
static test(contingencyTable) {
if (contingencyTable.length < 2 || contingencyTable[0].length < 2) {
return {
statistic: 0,
pValue: 1,
degreesOfFreedom: 0,
interpretation: 'Need at least 2x2 table for chi-squared test',
cramersV: 0,
};
}
const rows = contingencyTable.length;
const cols = contingencyTable[0].length;
// Calculate row and column totals
const rowTotals = contingencyTable.map((row) => row.reduce((sum, val) => sum + val, 0));
const colTotals = Array(cols).fill(0);
let grandTotal = 0;
for (let i = 0; i < rows; i++) {
for (let j = 0; j < cols; j++) {
colTotals[j] += contingencyTable[i][j];
grandTotal += contingencyTable[i][j];
}
}
if (grandTotal === 0) {
return {
statistic: 0,
pValue: 1,
degreesOfFreedom: 0,
interpretation: 'Empty contingency table',
cramersV: 0,
};
}
// Calculate expected frequencies and chi-squared statistic
let chiSquared = 0;
let lowExpectedCount = 0;
for (let i = 0; i < rows; i++) {
for (let j = 0; j < cols; j++) {
const expected = (rowTotals[i] * colTotals[j]) / grandTotal;
if (expected < 5) {
lowExpectedCount++;
}
if (expected > 0) {
const observed = contingencyTable[i][j];
chiSquared += Math.pow(observed - expected, 2) / expected;
}
}
}
const degreesOfFreedom = (rows - 1) * (cols - 1);
// Check assumption: expected frequencies ≥ 5
const totalCells = rows * cols;
if (lowExpectedCount / totalCells > 0.2) {
return {
statistic: Number(chiSquared.toFixed(6)),
pValue: 1,
degreesOfFreedom,
interpretation: 'Chi-squared test assumptions violated: >20% of cells have expected frequency <5',
cramersV: 0,
};
}
// Approximate p-value using chi-squared distribution
// Critical values depend on degrees of freedom
let pValue;
if (degreesOfFreedom === 1) {
if (chiSquared < 3.84)
pValue = 0.1;
else if (chiSquared < 6.64)
pValue = 0.05;
else if (chiSquared < 10.83)
pValue = 0.01;
else
pValue = 0.001;
}
else if (degreesOfFreedom === 2) {
if (chiSquared < 5.99)
pValue = 0.1;
else if (chiSquared < 9.21)
pValue = 0.05;
else if (chiSquared < 13.82)
pValue = 0.01;
else
pValue = 0.001;
}
else {
// General approximation
const criticalValue = degreesOfFreedom + 2 * Math.sqrt(2 * degreesOfFreedom);
if (chiSquared < criticalValue * 0.8)
pValue = 0.1;
else if (chiSquared < criticalValue)
pValue = 0.05;
else if (chiSquared < criticalValue * 1.3)
pValue = 0.01;
else
pValue = 0.001;
}
// Calculate Cramer's V (effect size)
const cramersV = Math.sqrt(chiSquared / (grandTotal * Math.min(rows - 1, cols - 1)));
const interpretation = pValue > 0.05
? 'No significant association between variables (p > 0.05)'
: 'Significant association between variables (p ≤ 0.05)';
return {
statistic: Number(chiSquared.toFixed(6)),
pValue: Number(pValue.toFixed(4)),
degreesOfFreedom,
interpretation,
cramersV: Number(cramersV.toFixed(4)),
};
}
}
exports.ChiSquaredTest = ChiSquaredTest;
/**
* Correlation significance test
* Tests if a correlation coefficient is significantly different from zero
*/
class CorrelationSignificanceTest {
static test(correlation, sampleSize) {
if (sampleSize < 3) {
return {
pValue: 1,
interpretation: 'Insufficient sample size for correlation significance test',
};
}
if (Math.abs(correlation) >= 1) {
return {
pValue: correlation === 0 ? 1 : 0,
interpretation: correlation === 0 ? 'Perfect zero correlation' : 'Perfect correlation',
};
}
// Calculate t-statistic for correlation
const tStatistic = correlation * Math.sqrt((sampleSize - 2) / (1 - correlation * correlation));
const degreesOfFreedom = sampleSize - 2;
// Approximate p-value using t-distribution
const absT = Math.abs(tStatistic);
let pValue;
if (degreesOfFreedom >= 30) {
// Large sample approximation (normal distribution)
if (absT < 1.96)
pValue = 0.1;
else if (absT < 2.58)
pValue = 0.05;
else if (absT < 3.29)
pValue = 0.01;
else
pValue = 0.001;
}
else {
// Small sample (t-distribution approximation)
if (absT < 2.0)
pValue = 0.1;
else if (absT < 2.5)
pValue = 0.05;
else if (absT < 3.5)
pValue = 0.01;
else
pValue = 0.001;
}
const interpretation = pValue > 0.05
? 'Correlation not significantly different from zero (p > 0.05)'
: 'Correlation significantly different from zero (p ≤ 0.05)';
return {
pValue: Number(pValue.toFixed(4)),
interpretation,
};
}
}
exports.CorrelationSignificanceTest = CorrelationSignificanceTest;
//# sourceMappingURL=statistical-tests.js.map