als-statistics/index.mjs

A powerful and lightweight JavaScript library for descriptive statistics, regression, clustering, outlier detection, and noise analysis using a flexible table/column architecture.

class ColumnFilter {
   constructor() {
      this.filterState = 0;
      this.init()
   }
   init() {
      this.filtered = {};
      this.maxFilter = 0;
      this.count = 0
   }
   get isFiltered() { return this.count > 0 }
   filterRows(indexes = []) {
      const { count } = this
      for (let i of indexes) {
         if (this.filtered[i]) continue
         this.filtered[i] = true
         this.count++
      }
      if (indexes[indexes.length - 1] > this.maxFilter) this.maxFilter = indexes[indexes.length - 1]
      if (this.count > count) this.filterState++
      return this
   }
   clearRowsFilters(indexes = []) {
      if (this.maxFilter === 0) return
      const { count, maxFilter } = this
      for (let i of indexes) {
         if (!this.filtered[i]) continue
         delete this.filtered[i]
         this.count--
      }
      if (!this.filtered[maxFilter]) { // If `maxFilter` deleted, find new max
         const keys = Object.keys(this.filtered).map(n => Number(n))
         this.maxFilter = keys[keys.length - 1] || 0;
      }
      if (count > this.count) this.filterState++
      return this
   }
   clearAllRowsFilters() { this.init(); this.filterState++; return this }
}
function compute(fn, { n, columns }) {
   const values = [];
   let j = 0;
   for (let i = 0; i < n; i++) {
      const obj = {}
      for (let colName in columns) {
         obj[colName] = columns[colName]._values[i]
      }
      values.push(fn(obj, j))
      j++
   }
   return values
}
function getType(values=[]) {
   if (!Array.isArray(values)) throw new Error("Input must be an array");
   let type = Number;
   for(let v of values) {
      if(typeof v === 'number') {
         if (!Number.isFinite(v) || isNaN(v)) throw new Error("All elements in the array must be finite valid numbers");
         continue
      }
      type = String
      break;
   }
   return type
}
class BaseColumn {
   cache = {};
   constructor(values=[], columnFilter, type) {
      if (!Array.isArray(values)) throw new Error("Input must be a non-empty array");
      this.columnFilter = columnFilter || new ColumnFilter()
      this.type = type || getType(values)
      this._values = values;
   }
   count(fn,filtered=true) { return (filtered ? this.values : this._values).filter(fn) }
   cached(name, fn) {
      const state = this.columnFilter.filterState;
      if (this.cache[name] && this.cache[name].state === state) return this.cache[name].value;
      this.cache[name] = { state, value: fn() };
      return this.cache[name].value;
   }
   filterRows(indexes) { this.columnFilter.filterRows(indexes) }
   clearRowsFilters(indexes) { this.columnFilter.clearRowsFilters(indexes) }
   clearAllRowsFilters() { this.columnFilter.clearAllRowsFilters() }
   filterRowsBy(fn) {
      const indexes = this._values.map((v, i) => fn(v, i) ? i : false).filter(v => v !== false)
      this.filterRows(indexes) 
   }
   get values() {
      return this.cached('values', () => {
         if (!this.columnFilter.isFiltered) return this._values
         return this._values.filter((v, i) => !this.columnFilter.filtered[i])
      })
   }
   get n() { return this.values.length }
}
class Column extends BaseColumn {
   constructor(values, columnFilter) {
      super(values, columnFilter)
   }
   clone(filtered=true,table) { return new this.constructor(filtered ? this.values : this._values,table) }
   get frequencies() { // Absolute frequencies for each unique value
      return this.cached('absoluteFrequencies', () => {
         const freq = new Map();
         for (const v of this.values) { freq.set(v, (freq.get(v) || 0) + 1) }
         return Object.fromEntries(freq);
      });
   }
   get relativeFrequencies() { // Relative frequencies (the proportion of each value relative to the total)
      return this.cached('relativeFrequencies', () => {
         const { frequencies, n } = this, relFreq = {};
         for (const key in frequencies) { relFreq[key] = frequencies[key] / n }
         return relFreq;
      });
   }
   get sorted() {
      return this.cached('sorted', () => {
         const valuesCopy = this.values.slice();
         if (valuesCopy.length === 0) return valuesCopy;
         if (this.type === Number) return valuesCopy.sort((a, b) => a - b);
         return valuesCopy.sort((a, b) => String(a).localeCompare(String(b)));
      });
   }
   percentile(p) {
      const { sorted, n } = this
      if (p < 0 || p > 100) throw new Error("Percentile must be between 0 and 100");
      if (p === 0) return sorted[0];
      if (p === 100) return sorted[n - 1];
      const index = Math.floor((p / 100) * (n - 1)); // Берем нижний индекс
      return sorted[index]; // Возвращаем число без усреднения
   }
   get median() { return this.cached('median', () => this.percentile(50)) }
   get q1() { return this.cached('q1', () => this.percentile(25)) }
   get q3() { return this.cached('q3', () => this.percentile(75)) }
}
class MovingAverage {
   constructor(sample, windowSize,sampleThreshold = 1000) {
      this.sample = sample
      this.windowSize = windowSize
      if (!Number.isInteger(windowSize) || windowSize < 1) throw new Error("windowSize must be a positive integer");
      if (windowSize > sample.n) throw new Error('window size cant be bigger than sample size')
      this.result = []
      sample.n > sampleThreshold ? this.bigSample() : this.naiveSample()
   }
   bigSample() {
      const { sample: { values, n }, windowSize,result } = this
      const prefixSums = new Array(n + 1).fill(0); 
      for (let i = 0; i < n; i++) { prefixSums[i + 1] = prefixSums[i] + values[i] }
      for (let i = 0; i + windowSize <= n; i++) {
         result.push((prefixSums[i + windowSize] - prefixSums[i]) / windowSize);
      }
   }
   naiveSample() {
      const { sample: { values, n }, windowSize,result } = this
      for (let start = 0; start + windowSize <= n; start++) {
         let sumWindow = 0;
         for (let i = start; i < start + windowSize; i++) { sumWindow += values[i] }
         result.push(sumWindow / windowSize);
      }
   }
}
class Noice {
   constructor(sample) {
      this.sample = sample
      this.zThreshold = 2.5 + (0.5 * Math.abs(sample.skewness)); // Dynamic z threshold depends on skewness
   }
   get noiseByRelativeDispersion() { return this.sample.relativeDispersion < 1 }
   get noiseByCV() { return this.sample.cv < 1 }
   get noiseByIQR() { return this.sample.iqr < this.sample.stdDev }
   get noiseBySkewness() { return Math.abs(this.sample.skewness) < 1 }
   noiseByZ(min = 0.01) {
      const { zScores, zThreshold, n, relativeDispersion } = this.sample
      const outliersAbove = zScores.filter(z => z > zThreshold).length / n;
      const outliersBelow = zScores.filter(z => z < -zThreshold).length / n;
      return (outliersAbove + outliersBelow) < Math.min(min, min * relativeDispersion);
   }
}
function confidenceInterval(sample) {
   const tTable95 = {
      1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571,
      6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228,
      11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.131,
      16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086,
      21: 2.080, 22: 2.074, 23: 2.069, 24: 2.064, 25: 2.060,
      26: 2.056, 27: 2.052, 28: 2.048, 29: 2.045, 30: 2.042
   };
   const { n, mean, stdDevSample } = sample;
   if (n < 2) return { low: mean, high: mean, width: 0 };
   const df = n - 1;
   let criticalValue = df <= 30 ? tTable95[df] : 1.96;
   const margin = criticalValue * (stdDevSample / Math.sqrt(n)); // Если стандартное отклонение равно 0, margin будет равен 0
   const low = mean - margin;
   const high = mean + margin;
   const width = mean !== 0 ? ((high - low) / Math.abs(mean)) * 100 : (high - low); // Если mean равен 0, нельзя вычислять относительную ширину делением на mean.
   return { low, high, width };
}
function mode(values) {
   const freq = new Map();
   let maxFreq = 0;
   for (let num of values) {
      const count = (freq.get(num) || 0) + 1;
      freq.set(num, count);
      if (count > maxFreq) maxFreq = count;
   }
   const modes = [];
   for (const [val, count] of freq.entries()) {
      if (count === maxFreq) modes.push(val);
   }
   return modes.length === values.length ? [] : modes;
}
function outliersZScore({ zScores, values },threshold = 3, twoFactors = true) {
   const outliers = [], indexes = [], zs = [];
   for (let i = 0; i < zScores.length; i++) {
      const z = twoFactors ? Math.abs(zScores[i]) : zScores[i]
      if (z <= threshold) continue
      outliers.push(values[i]);
      indexes.push(i)
      zs.push(zScores[i])
   }
   return { values: outliers, indexes, zScores: zs };
}
function weightedMean(weights,values) {
   if (!Array.isArray(weights) || weights.length !== values.length) throw new Error("Weights must be an array of the same length as values");
   let wSum = 0, valSum = 0;
   for (let i = 0; i < values.length; i++) {
      wSum += weights[i];
      valSum += values[i] * weights[i];
   }
   return wSum === 0 ? 0 : valSum / wSum;
}

const EPS = 3e-14;
const FPMIN = 1e-30;
const P = [676.5203681218851, -1259.1392167224028, 771.32342877765313, -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6, 1.5056327351493116e-7];
const G = 7;
function betacfNR(x, a, b) {
   const MAXIT = 200;
   let qab = a + b;
   let qap = a + 1;
   let qam = a - 1; // используется только для инициализации, но оставим для ясности
   let c = 1.0;
   let d = 1.0 - (qab * x) / qap;
   if (Math.abs(d) < FPMIN) d = FPMIN;
   d = 1.0 / d;
   let h = d;
   for (let m = 1; m <= MAXIT; m++) {
      const m2 = 2 * m;
      // Even step
      let aa = m * (b - m) * x / ((a + m2 - 1) * (a + m2));
      d = 1.0 + aa * d;
      if (Math.abs(d) < FPMIN) d = FPMIN;
      c = 1.0 + aa / c;
      if (Math.abs(c) < FPMIN) c = FPMIN;
      d = 1.0 / d;
      h *= (d * c);
      // Odd step
      aa = - (a + m) * (qab + m) * x / ((a + m2) * (a + m2 + 1));
      d = 1.0 + aa * d;
      if (Math.abs(d) < FPMIN) d = FPMIN;
      c = 1.0 + aa / c;
      if (Math.abs(c) < FPMIN) c = FPMIN;
      d = 1.0 / d;
      let delta = d * c;
      h *= delta;
      if (Math.abs(delta - 1.0) < EPS) {
         break;
      }
   }
   return h;
}
function gammaLn(z) {
   if (z < 0.5) return Math.log(Math.PI) - Math.log(Math.sin(Math.PI * z)) - gammaLn(1 - z);
   else {
      z -= 1;
      let x = 0.99999999999980993;
      for (let i = 0; i < P.length; i++) {
         x += P[i] / (z + i + 1);
      }
      const t = z + G + 0.5;
      return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x);
   }
}
function betaLn(a, b) { return gammaLn(a) + gammaLn(b) - gammaLn(a + b) }
function regularizedIncompleteBeta(x, a, b) {
   if (x <= 0) return 0;
   if (x >= 1) return 1;
   const bt = Math.exp(a * Math.log(x) + b * Math.log(1 - x) - betaLn(a, b));
   let sym = false;
   let xx = x, aa = a, bb = b;
   if (x > (a + 1) / (a + b + 2)) {
      sym = true;
      xx = 1 - x;
      aa = b;
      bb = a;
   }
   const cf = betacfNR(xx, aa, bb);
   let val = bt / aa * cf;
   if (sym) val = 1 - val;
   return val;
}
function tCDF(t, df) {
   if (df <= 0) throw new Error("Degrees of freedom must be positive");
   if (t === 0) return 0.5;
   const x = df / (df + t * t);
   const a = df / 2;
   const b = 0.5;
   const ibeta = regularizedIncompleteBeta(x, a, b);
   return (t >= 0) ? 1 - 0.5 * ibeta : 0.5 * ibeta;
}
class PearsonP {
   constructor(n, cov,sigX, sigY) {
      this.r = (sigX === 0 || sigY === 0) ? 0 : cov / (sigX * sigY)
      this.df = n - 2;
      this.t = 0;
      this.p = 1;
      if (this.df > 1) this.calculateP(this)
   }
   get significant() { return this.p < 0.0001 }
   get sig() { return (this.p < 0.0001) ? "<0.0001" : this.p.toFixed(4) }
   calculateP({ r, df }) {
      this.t = r * Math.sqrt(df / Math.max(1 - r * r, 1e-16));
      this.p = 2 * (1 - tCDF(Math.abs(this.t), df));
      if (this.p < 0) this.p = 0;
      if (this.p > 1) this.p = 1;
   }
}
class Comparative {
   constructor(sample1, sample2) {
      if (sample1.n !== sample2.n) throw new Error("Length of samples must match");
      this.sample1 = sample1; this.sample2 = sample2;
      this.n = this.sample1.n
   }
   get sumCov() {
      const { sample1: { mean: m1, values: values1 }, sample2: { mean: m2, values: values2 }, n } = this
      let sumCov = 0;
      for (let i = 0; i < n; i++) { sumCov += (values1[i] - m1) * (values2[i] - m2) }
      return sumCov
   }
   get covariancePopulation() { return this.sumCov / this.n }
   get covarianceSample() { return this.n < 2 ? 0 : this.sumCov / (this.n - 1) }
   get correlationPopulation() { return this.pearsonPopulation.r }
   get correlationSample() { return this.pearsonSample.r }
   get pearsonPopulation() {
      return new PearsonP(this.n, this.covariancePopulation, this.sample1.stdDevPopulation, this.sample2.stdDevPopulation)
   }
   get pearsonSample() {
      return new PearsonP(this.n, this.covarianceSample, this.sample1.stdDevSample, this.sample2.stdDevSample)
   }
   twoSampleTTest() {
      const { n, sample1: { mean: m1, varianceSample: var1 }, sample2: { mean: m2, varianceSample: var2 } } = this
      const pooledVariance = ((n - 1) * var1 + (n - 1) * var2) / (2 * n - 2);
      const sp = Math.sqrt(pooledVariance);
      const t = (m1 - m2) / (sp * Math.sqrt(2 / n));
      const df = 2 * n - 2;
      const F = var1 / var2;
      return { t, df, F };
   }
}
class RatioColumn extends Column {
   constructor(values, columnFilter) {
      super(values, columnFilter, Number)
   }
   get sum() { return this.cached('sum', () => this.values.reduce((acc, val) => acc + val, 0)) }
   get mean() { return this.cached('mean', () => this.sum / this.n) }
   get sorted() { return this.cached('sorted', () => this.values.slice().sort((a, b) => a - b)) }
   get min() { return this.cached('min', () => this.sorted[0]) }
   get max() { return this.cached('max', () => this.sorted[this.n - 1]) }
   get range() { return this.cached('range', () => this.max - this.min) }
   get variance() { return this.variancePopulation }
   get stdDev() { return this.stdDevPopulation }
   get skewness() { return this.skewnessPopulation }
   get kurtosis() { return this.kurtosisPopulation }
   get cv() { return this.cached('cv', () => this.mean === 0 ? 0 : this.stdDevPopulation / this.mean) } // Coefficient of Variation - stdDevPopulation / mean
   get iqr() { return this.cached('iqr', () => this.q3 - this.q1) }
   outliersZScore(threshold = 3, twoFactors = true) { return outliersZScore(this, threshold, twoFactors) }
   weightedMean(weights) { return weightedMean(weights, this.values) }
   noice() { return new Noice(this) }
   ma(windowSize) { return new MovingAverage(this, windowSize).result }
   get variancePopulation() {
      return this.cached('variancePopulation', () => {
         return this.values.reduce((acc, val) => acc + Math.pow(val - this.mean, 2), 0) / this.n;
      });
   }
   get stdDevPopulation() { return this.cached('stdDevPopulation', () => Math.sqrt(this.variancePopulation)) }
   get zScores() {
      return this.cached('zScores', () => {
         const { mean, stdDevPopulation, values } = this;
         if (stdDevPopulation === 0) return values.map(v => 0);
         return values.map(v => (v - mean) / stdDevPopulation);
      });
   }
   get skewnessPopulation() {
      return this.cached('skewnessPopulation', () => {
         const { values, mean, stdDevPopulation, n } = this;
         if (stdDevPopulation === 0) return 0; // If stdDev = 0
         return values.reduce((acc, v) => acc + ((v - mean) / stdDevPopulation) ** 3, 0) / n;
      });
   }
   get kurtosisPopulation() {
      return this.cached('kurtosisPopulation', () => {
         const sumZ4 = this.zScores.reduce((acc, z) => acc + z ** 4, 0);
         return (sumZ4 / this.n) - 3;
      });
   }
   get varianceSample() { // Sample (selected): Bessel's correction. s^2 = (1/(n-1)) * sum( (x - mean)^2 )
      return this.cached('varianceSample', () => {
         if (this.n < 2) return 0;
         return this.values.reduce((acc, val) => acc + (val - this.mean) ** 2, 0) / (this.n - 1);
      });
   }
   get stdDevSample() { return this.cached('stdDevSample', () => Math.sqrt(this.varianceSample)) }
   get relativeDispersion() { return this.median === 0 ? 0 : this.stdDev / this.median }
   get skewnessSample() {
      return this.cached('skewnessSample', () => {
         if (this.n < 3) return 0;
         const { mean, values, n, stdDevSample } = this;
         const numerator = values.reduce((acc, val) => acc + ((val - mean) / stdDevSample) ** 3, 0);
         const factor = n / ((n - 1) * (n - 2));
         return factor * numerator;
      });
   }
   get kurtosisSample() {
      return this.cached('kurtosisSample', () => {
         if (this.n < 4) return 0;
         const { mean, values, n, stdDevSample } = this;
         const z4sum = values.reduce((acc, val) => acc + Math.pow((val - mean) / stdDevSample, 4), 0);
         const a = (n * (n + 1)) / ((n - 1) * (n - 2) * (n - 3));
         const b = 3 * Math.pow(n - 1, 2) / ((n - 2) * (n - 3));
         return a * z4sum - b;
      });
   }
   get mode() { return this.cached('mode', () => mode(this.values)) }
   get normalizedValues() { // Min-Max scaling
      return this.cached('normalizedValues', () => {
         const { min, range, values } = this;
         if (range === 0) return values.map(() => 0);
         return values.map(v => (v - min) / range);
      });
   }
   get geometricMean() { // GM = (П(i=1..n) x_i)^(1/n). Only for x_i > 0.
      return this.cached('geometricMean', () => {
         if (!this.values.every(v => v > 0)) return NaN;
         const logSum = this.values.reduce((acc, v) => acc + Math.log(v), 0);
         return Math.exp(logSum / this.n);
      });
   }
   get harmonicMean() { // HM = n / Σ(1/x_i) . Only for x_i > 0
      return this.cached('harmonicMean', () => {
         if (!this.values.every(v => v > 0)) return NaN;
         const denom = this.values.reduce((acc, v) => acc + 1 / v, 0);
         return this.n / denom;
      });
   }
   get flatness() { return this.cached('flatness', () => this.geometricMean / this.mean) }
   get sumOfSquares() { return this.cached('sumOfSquares', () => this.values.reduce((acc, v) => acc + v ** 2, 0)) }  // Сумма квадратов (энергетическая норма). Σ(x_i^2)
   get confidenceInterval95() { return this.cached('confidenceInterval95', () => confidenceInterval(this)) }
   get outliersIQR() {
      const { q1, q3, iqr, values } = this;
      const lowerBound = q1 - 1.5 * iqr, upperBound = q3 + 1.5 * iqr;
      return values.filter(v => v < lowerBound || v > upperBound);
   }
   get noiseStability() {
      return this.cached('noiseStability', () => {
         const { values, mean, n } = this;
         if (n < 2) return Array(n).fill(0);
         const variance = values.reduce((acc, v) => acc + (v - mean) ** 2, 0) / n;
         const stdDev = Math.sqrt(variance);
         return stdDev;
      });
   }
   get spectralPowerDensityArray() {
      return this.cached('spectralPowerDensityArray', () => {
         const { values, sumOfSquares } = this;
         if (sumOfSquares === 0) return values.map(() => 0);
         return values.map(v => (v ** 2) / sumOfSquares);
      });
   }
   get spectralPowerDensityMetric() {
      return this.cached('spectralPowerDensityMetric', () => {
         const spdArray = this.spectralPowerDensityArray;
         const geometricMean = Math.exp(spdArray.reduce((sum, x) => sum + Math.log(x + Number.EPSILON), 0) / spdArray.length);
         const arithmeticMean = spdArray.reduce((sum, x) => sum + x, 0) / spdArray.length;
         return geometricMean / arithmeticMean; // Близкое к 1 → равномерный шум, ближе к 0 → голос
      });
   }
   get xValues() { return this.cached('xValues', () => Array.from({ length: this.n }, (_, i) => i + 1)) }
   regressionSlope(customX = null) {
      const x = customX || this.xValues;
      if (!Array.isArray(x) || x.length !== this.n) throw new Error("x must be an array of same length as values");
      const xCol = new RatioColumn(x);
      const cov = new Comparative(xCol, this).covarianceSample;
      const varX = xCol.varianceSample;
      return varX === 0 ? 0 : cov / varX;
   }
}
function newColumn(values,table) { 
   return getType(values) === Number 
      ? new RatioColumn(values, table, Number) 
      : new Column(values, table, String) 
}
function extractMetric(col, metricName) {
   const keys = metricName.split('.')
   let value = col;
   for (let i = 0; i < keys.length; i++) {
      if (value === undefined) break;
      value = value[keys[i]];
      if (Array.isArray(value) && value.length) value = newColumn(value)
   }
   return value;
}
function extractMetrics(columns,metricName) {
   return Object.values(columns)
      .map(col => extractMetric(col, metricName))
      .filter(v => v !== undefined)
}
function transpose(table, { n, columns, filtered, isFiltered }) {
   let $columns = Object.values(columns).map(col => col.type === Number ? col.values : null)
   let j = 0;
   for (let i = 0; i < n; i++) {
      if (isFiltered && filtered[i]) continue
      table.addColumn(j, $columns.map(values => values[j]))
      j++
   }
   return table
}
function filterColumns(columns, columnFilter) {
   const includeNames = [], excludeNames = [], regexPatterns = [];
   if (Array.isArray(columnFilter)) { // Разделяем фильтры
      columnFilter.forEach(filter => {
         if (typeof filter === 'number') includeNames.push(filter)
         else if (typeof filter === "string") {
            if (filter.startsWith("-")) excludeNames.push(filter.slice(1));
            else includeNames.push(filter);
         } else if (filter instanceof RegExp) {
            regexPatterns.push(filter);
         }
      });
   }
   const columnNames = Object.keys(columns);
   let filteredNames = columnNames;
   if (includeNames.length > 0) filteredNames = includeNames; // Если есть явные имена, берём только их
   else if (regexPatterns.length > 0) { // Если есть regex, фильтруем по ним
      filteredNames = columnNames.filter(name =>
         regexPatterns.some(pattern => pattern.test(name))
      );
   }
   if (excludeNames.length > 0) filteredNames = filteredNames.filter(name => !excludeNames.includes(name)); // Применяем исключения
   return filteredNames;
}
function addRow(obj, index = null, table) {
   // Проверка соответствия ключей и типов
   const colNames = Object.keys(table.columns);
   for (let key of Object.keys(obj)) {
      if (!colNames.includes(key)) return false;
      const colType = table.columns[key].type;
      if (colType === Number && typeof obj[key] !== 'number') return false;
      if (colType === String && typeof obj[key] !== 'string') return false;
   }
   for (let colName of colNames) {
      if (!(colName in obj)) return false;
   }
   if (index === null || index >= table.n) { // Добавление строки
      for (let [colName, value] of Object.entries(obj)) { // Добавление в конец
         table.columns[colName]._values.push(value);
      }
      table.n++;
   } else { // Вставка по индексу
      for (let [colName, value] of Object.entries(obj)) {
         table.columns[colName]._values.splice(index, 0, value);
      }
      table.n++;
      // Обновление фильтра
      const updatedFiltered = {};
      for (let i in table.filtered) {
         const numIndex = Number(i);
         if (numIndex >= index) updatedFiltered[numIndex + 1] = true;
         else updatedFiltered[numIndex] = true;
      }
      table.filtered = updatedFiltered;
      if (table.maxFilter >= index) table.maxFilter++;
   }
   table.filterState++;
   return true;
}
class CronbachAlpha {
   static cronbachAlpha(table, obj = {}) {
      const { columnsN } = table;
      if (columnsN < 2) throw new Error("For Alpha Cronbach needs >=2 ratio columns.");
      obj.sumOfVariances = table.descriptive('varianceSample').sum;
      obj.sumColumnVariance = table.compute(row => Object.values(row).reduce((s, v) => s + v, 0)).varianceSample;
      obj.bessel = columnsN / (columnsN - 1);
      obj.alpha = obj.sumColumnVariance === 0 ? 0 : obj.bessel * (1 - obj.sumOfVariances / obj.sumColumnVariance);
      return obj;
   }
   constructor(table) {
      this.table = table;
      CronbachAlpha.cronbachAlpha(table, this);
   }
   get perColumn() { return this.ifItemsDeleted(); }
   ifItemsDeleted(columnFilter) {
      const result = {};
      for (let colName in this.table.columns) {
         const filter = columnFilter || ['-' + colName];
         result[colName] = CronbachAlpha.cronbachAlpha(this.table.clone(true, filter)).alpha;
      }
      return result;
   }
}
class Dbscan {
   constructor(table, eps = 0.4, minPts = 3) {
      if (table.columnsN < 2) throw new Error('2 or more columns required')
      this.table = table;
      this.eps = eps;
      this.minPts = minPts;
      this.columnsArray = Object.entries(table.columns);
      this.n = this.columnsArray.length;
      this.labels = new Array(this.n).fill(0); // 0: не обработан, -1: шум, 1+: кластер
      this.clusters = [];
      this.distances = null;
      this.computeDistances();
      this.run();
      this.buildClusters();
   }
   computeDistances() { // Вычисление матрицы расстояний
      this.distances = new Array(this.n).fill(null).map(() => new Array(this.n).fill(null));
      for (let i = 0; i < this.n; i++) {
         for (let j = i + 1; j < this.n; j++) {
            const { correlationSample } = this.table.compare(this.columnsArray[i][0], this.columnsArray[j][0]);
            const dist = 1 - correlationSample;
            this.distances[i][j] = dist;
            this.distances[j][i] = dist;
         }
         this.distances[i][i] = 0; // расстояние до себя = 0
      }
   }
   findNeighbors(pointIdx) { // Поиск соседей для точки
      const neighbors = [];
      for (let i = 0; i < this.n; i++) {
         if (this.distances[pointIdx][i] <= this.eps) {
            neighbors.push(i);
         }
      }
      return neighbors;
   }
   expandCluster(pointIdx, clusterId) { // Расширение кластера
      this.labels[pointIdx] = clusterId;
      const seeds = [...this.findNeighbors(pointIdx).filter(idx => idx !== pointIdx)];
      while (seeds.length > 0) {
         const current = seeds.pop();
         if (this.labels[current] === -1) this.labels[current] = clusterId; // шум в кластер
         if (this.labels[current] !== 0) continue; // уже обработан
         this.labels[current] = clusterId;
         const currentNeighbors = this.findNeighbors(current);
         if (currentNeighbors.length >= this.minPts) {
            seeds.push(...currentNeighbors.filter(idx => this.labels[idx] === 0));
         }
      }
   }
   run() { // Основной алгоритм DBSCAN
      let clusterId = 0;
      for (let i = 0; i < this.n; i++) {
         if (this.labels[i] !== 0) continue; // уже обработан
         const neighbors = this.findNeighbors(i);
         if (neighbors.length < this.minPts) {
            this.labels[i] = -1; // шум
            continue;
         }
         clusterId++;
         this.expandCluster(i, clusterId);
      }
   }
   buildClusters() { // Формирование кластеров как таблиц
      for (let id = 1; id <= Math.max(...this.labels); id++) {
         const clusterTable = new this.table.Table();
         this.columnsArray.forEach(([name, column], idx) => {
            if (this.labels[idx] === id) {
               clusterTable.addColumn(name, column.clone(true));
            }
         });
         if (clusterTable.columnsN > 0) this.clusters.push(clusterTable);
      }
   }
}
class Hdbscan {
   constructor(table, minClusterSize = 2) {
      if (table.columnsN < 2) throw new Error('2 or more columns required');
      this.table = table;
      this.minClusterSize = minClusterSize;
      this.columnsArray = Object.entries(table.columns);
      this.n = this.columnsArray.length;
      this.labels = new Array(this.n).fill(-1);
      this.clusters = [];
      this.distances = null;
      this.mreachDistances = null;
      this.mst = [];
      this.hierarchy = [];
      this.computeDistances();
      this.computeMutualReachability();
      this.buildMST();
      this.buildHierarchy();
      this.extractClusters();
      this.buildClusters();
   }
   computeDistances() {
      this.distances = new Array(this.n).fill(null).map(() => new Array(this.n).fill(null));
      for (let i = 0; i < this.n; i++) {
         for (let j = i + 1; j < this.n; j++) {
            const { correlationSample } = this.table.compare(this.columnsArray[i][0], this.columnsArray[j][0]);
            const dist = isNaN(correlationSample) ? 1 : 1 - correlationSample;
            this.distances[i][j] = dist;
            this.distances[j][i] = dist;
         }
         this.distances[i][i] = 0;
      }
   }
   computeMutualReachability() {
      this.mreachDistances = new Array(this.n).fill(null).map(() => new Array(this.n).fill(null));
      const coreDistances = new Array(this.n);
      for (let i = 0; i < this.n; i++) {
         const distances = this.distances[i].slice().sort((a, b) => a - b);
         coreDistances[i] = distances[Math.min(this.minClusterSize - 1, this.n - 1)];
      }
      for (let i = 0; i < this.n; i++) {
         for (let j = i; j < this.n; j++) {
            const mreach = Math.max(coreDistances[i], coreDistances[j], this.distances[i][j]);
            this.mreachDistances[i][j] = mreach;
            this.mreachDistances[j][i] = mreach;
         }
      }
   }
   buildMST() {
      const visited = new Array(this.n).fill(false);
      const key = new Array(this.n).fill(Infinity);
      const parent = new Array(this.n).fill(-1);
      key[0] = 0;
      for (let count = 0; count < this.n - 1; count++) {
         let minKey = Infinity;
         let u = -1;
         for (let v = 0; v < this.n; v++) {
            if (!visited[v] && key[v] < minKey) {
               minKey = key[v];
               u = v;
            }
         }
         visited[u] = true;
         if (parent[u] !== -1) this.mst.push([parent[u], u, this.mreachDistances[parent[u]][u]]);
         for (let v = 0; v < this.n; v++) {
            if (!visited[v] && this.mreachDistances[u][v] < key[v]) {
               key[v] = this.mreachDistances[u][v];
               parent[v] = u;
            }
         }
      }
      this.mst.sort((a, b) => a[2] - b[2]);
   }
   buildHierarchy() {
      const uf = new UnionFind(this.n);
      const clusterSizes = new Array(this.n).fill(1);
      let clusterId = 0; // Начинаем с 0 для уникальных кластеров
      this.hierarchy = [];
      for (let i = 0; i < this.n; i++) {
         this.hierarchy.push({ clusterId: i, lambdaBirth: 0, lambdaDeath: Infinity, points: [i], size: 1 });
      }
      for (const [p, q, weight] of this.mst) {
         const lambda = 1 / weight;
         const cp = uf.find(p);
         const cq = uf.find(q);
         if (cp !== cq) {
            const sizeP = clusterSizes[cp];
            const sizeQ = clusterSizes[cq];
            uf.union(cp, cq);
            const newRoot = uf.find(cp);
            clusterSizes[newRoot] = sizeP + sizeQ;
            // Обновляем lambdaDeath для старых кластеров
            this.hierarchy[cp].lambdaDeath = lambda;
            this.hierarchy[cq].lambdaDeath = lambda;
            // Добавляем новый кластер
            this.hierarchy.push({
               clusterId: clusterId + this.n,
               lambdaBirth: lambda,
               lambdaDeath: Infinity,
               points: this.hierarchy[cp].points.concat(this.hierarchy[cq].points),
               size: sizeP + sizeQ
            });
            clusterId++;
         }
      }
   }
   extractClusters() {
      // Вычисляем стабильность каждого кластера
      const stability = new Map();
      this.hierarchy.forEach(cluster => {
         if (cluster.size >= this.minClusterSize) {
            const stabilityValue = cluster.size * (cluster.lambdaDeath - cluster.lambdaBirth);
            stability.set(cluster.clusterId, stabilityValue);
         }
      });
      // Извлекаем кластеры, сравнивая стабильность
      const activeClusters = new Map();
      this.hierarchy.forEach(cluster => {
         if (cluster.size < this.minClusterSize) return;
         let isStable = true;
         const childrenStability = cluster.points.reduce((sum, p) => {
            const child = this.hierarchy.find(c => c.clusterId === p);
            return sum + (stability.get(child.clusterId) || 0);
         }, 0);
         if (childrenStability > stability.get(cluster.clusterId)) {
            isStable = false;
         }
         if (isStable) {
            activeClusters.set(cluster.clusterId, cluster.points);
         }
      });
      // Присваиваем метки точкам
      let clusterId = 1;
      const assigned = new Set();
      activeClusters.forEach((points, id) => {
         if (points.length >= this.minClusterSize) {
            points.forEach(p => {
               if (!assigned.has(p)) {
                  this.labels[p] = clusterId;
                  assigned.add(p);
               }
            });
            clusterId++;
         }
      });
      // Отладка
      console.log('Points:', this.n);
      console.log('Labels:', this.labels);
      console.log('Clusters found:', new Set(this.labels.filter(l => l !== -1)).size);
   }
   buildClusters() {
      const maxLabel = Math.max(...this.labels);
      if (maxLabel < 1) {
         console.error('No clusters found');
         return;
      }
      for (let id = 1; id <= maxLabel; id++) {
         const clusterTable = new this.table.Table();
         this.columnsArray.forEach(([name, column], idx) => {
            if (this.labels[idx] === id) {
               clusterTable.addColumn(name, column.clone(true));
            }
         });
         if (clusterTable.columnsN > 0) this.clusters.push(clusterTable);
      }
   }
}
class UnionFind {
   constructor(n) {
      this.parent = new Array(n).fill(null).map((_, i) => i);
      this.rank = new Array(n).fill(0);
   }
   find(x) {
      if (this.parent[x] !== x) this.parent[x] = this.find(this.parent[x]);
      return this.parent[x];
   }
   union(x, y) {
      const px = this.find(x);
      const py = this.find(y);
      if (px === py) return;
      if (this.rank[px] < this.rank[py]) {
         this.parent[px] = py;
      } else if (this.rank[px] > this.rank[py]) {
         this.parent[py] = px;
      } else {
         this.parent[py] = px;
         this.rank[px]++;
      }
   }
}
const MatrixUtils = {
   transpose(A) {
      return A[0].map((_, i) => A.map(row => row[i]));
   },
   multiply(A, B) {
      const result = Array.from({ length: A.length }, () => Array(B[0].length).fill(0));
      for (let i = 0; i < A.length; i++) {
         for (let j = 0; j < B[0].length; j++) {
            for (let k = 0; k < B.length; k++) {
               result[i][j] += A[i][k] * B[k][j];
            }
         }
      }
      return result;
   },
   multiplyVec(A, b) {
      return A.map(row => row.reduce((acc, aij, j) => acc + aij * b[j], 0));
   },
   inverse(A) {
      const n = A.length;
      const I = Array.from({ length: n }, (_, i) => Array.from({ length: n }, (_, j) => (i === j ? 1 : 0)));
      const AI = A.map((row, i) => [...row, ...I[i]]);
      for (let i = 0; i < n; i++) {
         let maxRow = i;
         for (let k = i + 1; k < n; k++) {
            if (Math.abs(AI[k][i]) > Math.abs(AI[maxRow][i])) maxRow = k;
         }
         [AI[i], AI[maxRow]] = [AI[maxRow], AI[i]];
         const diag = AI[i][i];
         for (let j = 0; j < 2 * n; j++) AI[i][j] /= diag;
         for (let k = 0; k < n; k++) {
            if (k !== i) {
               const factor = AI[k][i];
               for (let j = 0; j < 2 * n; j++) AI[k][j] -= factor * AI[i][j];
            }
         }
      }
      return AI.map(row => row.slice(n));
   }
};
class LinearRegression {
   constructor(table, yName, xNames = []) {
      this.table = table;
      this.yName = yName;
      this.xNames = xNames
      this._originalX = xNames.length > 0 ? xNames : Object.keys(table.columns).filter(k => k !== yName);
      this._mediator = null;
      this._moderator = null;
      this._interactionName = null;
   }
   mediator(name) { this._mediator = name; return this }
   moderator(name) { this._moderator = name; return this }
   calculate() {
      const fullCols = { ...this.table.columns };
      this.xNames = [...this._originalX];
      if (this._mediator && !this.xNames.includes(this._mediator)) this.xNames.push(this._mediator); // Медиатор
      if (this._moderator) {
         const x = this.table.columns[this._originalX[0]].values;
         const z = this.table.columns[this._moderator].values;
         this._interactionName = `${this._originalX[0]}*${this._moderator}`;
         fullCols[this._interactionName] = { values: x.map((xi, i) => xi * z[i]) };
         if (!this.xNames.includes(this._moderator)) this.xNames.push(this._moderator);
         this.xNames.push(this._interactionName);
      }
      this.y = this.table.columns[this.yName].values;
      this.X = fullCols[this.xNames[0]].values.map((_, i) => [1, ...this.xNames.map(name => fullCols[name].values[i])]);
      this.n = this.X.length;
      this.k = this.X[0].length;
      this.coefficients = this.computeCoefficients();
      if (this.coefficients.some(c => !isFinite(c))) throw new Error("Regression failed: singular matrix or constant predictors");
      this.yHat = this.predict(this.X);
      this.residuals = this.y.map((yi, i) => yi - this.yHat[i]);
      this.r2 = this.computeR2();
      this.standardErrors = this.computeStandardErrors();
      this.pValues = this.computePValues();
      return this;
   }
   computeCoefficients() {
      const XT = MatrixUtils.transpose(this.X);
      const XTX = MatrixUtils.multiply(XT, this.X);
      const XTy = MatrixUtils.multiplyVec(XT, this.y);
      return MatrixUtils.solve ? MatrixUtils.solve(XTX, XTy) : MatrixUtils.multiplyVec(MatrixUtils.inverse(XTX), XTy);
   }
   predict(X) { return X.map(row => row.reduce((acc, val, j) => acc + val * this.coefficients[j], 0)) }
   computeR2() {
      const yMean = this.y.reduce((a, b) => a + b, 0) / this.n;
      const ssTot = this.y.reduce((acc, yi) => acc + (yi - yMean) ** 2, 0);
      const ssRes = this.residuals.reduce((acc, e) => acc + e ** 2, 0);
      return 1 - ssRes / ssTot;
   }
   computeStandardErrors() {
      const XT = MatrixUtils.transpose(this.X);
      const XTX = MatrixUtils.multiply(XT, this.X);
      const invXTX = MatrixUtils.inverse(XTX);
      const mse = this.residuals.reduce((acc, e) => acc + e ** 2, 0) / (this.n - this.k);
      return invXTX.map((row, i) => Math.sqrt(row[i] * mse));
   }
   computePValues() {
      return this.standardErrors.map((se, i) => {
         const t = this.coefficients[i] / se;
         return 2 * (1 - tCDF(Math.abs(t), this.n - this.k));
      });
   }
   get result() {
      if(!this.xNames) this.calculate()
      return {
         Variable: ['Intercept', ...this.xNames],
         Coefficient: this.coefficients,
         StdError: this.standardErrors,
         pValue: this.pValues
      };
   }
   get htmlTable() {
      const { Variable, Coefficient, StdError, pValue } = this.result
      return /*html*/`<table>
         <thead><tr style="text-align:left;">${['Variable', 'Coefficient', 'StdError', 'pValue'].map(v => /*html*/`<th>${v}</th>`).join('')}</tr></thead>
         <tbody>
            ${Variable.map((vName, i) => /*html*/`<tr>
               <th style="text-align:left;">${vName}</th>
               <td>${Coefficient[i].toFixed(3)}</td>
               <td>${StdError[i].toFixed(3)}</td>
               <td>${pValue[i].toFixed(5)}</td>
            </tr>`).join('')}
         </tbody>
      </table>`
   }
}

class Table extends ColumnFilter {
   constructor(data) {
      super()
      this.n = 0;
      this.columns = {}
      this.columnsN = 0
      if(data && typeof data === 'object' && !Array.isArray(data) && data !== null) {
         for (let colName in data) this.addColumn(colName, data[colName])
      }
   }
   get Table() { return Table }
   get cronbachAlpha() { return new CronbachAlpha(this) }
   addRow(obj, index) { return addRow(obj, index, this) }
   deleteColumn(name) {
      if (!this.columns[name]) return;
      this.columns[name].columnFilter = new ColumnFilter();
      delete this.columns[name];
      const colValues = Object.values(this.columns)
      this.columnsN = colValues.length
      this.n = Math.max(...colValues.map(col => col.n), 0);
      return this
   }
   addColumn(name, values = []) {
      this.columns[name] = values.constructor.name.includes('Column') ? values : newColumn(values, this)
      if (this.columns[name].n > this.n) this.n = this.columns[name].n
      this.columnsN++
      return this.columns[name]
   }
   filterRowsBy(colName, fn) { 
      if (this.columns[colName]) this.columns[colName].filterRowsBy(fn);
      return this
   }
   clone(filtered = true, columnFilter = null) {
      const table = new Table();
      const filteredNames = filterColumns(this.columns, columnFilter);
      for (let columnName of filteredNames) {
         if (!this.columns[columnName]) continue
         table.addColumn(columnName, this.columns[columnName].clone(filtered, table));
      }
      return table;
   }
   compare(colName1, colName2) { return new Comparative(this.columns[colName1], this.columns[colName2]) }
   transpose() { return transpose(new Table(), this) }
   descriptive(metricName, table) { return newColumn(extractMetrics(this.columns, metricName), table) }
   compute(fn, targetName) {
      return targetName ? this.addColumn(targetName, compute(fn, this)) : newColumn(compute(fn, this))
   }
   where(fn) { return compute((obj, i) => {if(!this.filtered[i] && fn(obj, i)) return i }, this).filter((v) => v !== undefined) }
   dbscan(eps = 0.4, minPts = 3) { return new Dbscan(this, eps, minPts) }
   hdbscan(minPts = 3) { return new Hdbscan(this, minPts) }
   linearRegression(yName, xNames) { return new LinearRegression(this,yName,xNames) }
   get json() {
      const obj = {}
      for(const colName in this.columns) {
         obj[colName] = this.columns[colName].values
      }
      return obj
   }
}
const range = (start, end, step = 1) => Array.from({ length: Math.floor((end - start) / step) }, (v, i) => start + i * step);
class Statistics {
   static range = range
   static statistics = new Statistics('default-statistics')
   static newTable(data) { return new Table(data) }
   static newColumn(values) { return newColumn(values) }
   constructor() {
      this.tables = {}
   }
   addTable(name,obj) { this.tables[name] = new Table(obj); return this.tables[name] }
   filterRows(indexes) { for (let n in this.tables) { this.tables[n].filterRows(indexes) }; return this }
   clearRowsFilters(indexes) { for (let n in this.tables) { this.tables[n].clearRowsFilters(indexes) }; return this }
   clearAllRowsFilters() { for (let n in this.tables) { this.tables[n].clearAllRowsFilters() }; return this }
   descriptive(metricName) {
      const newTable = new Table()
      Object.entries(this.tables).forEach(([name,table]) => {
         newTable.addColumn(name,table.descriptive(metricName,newTable))
      })
      return newTable
   }
   transpose() {
      const newStatistics = new Statistics()
      for (let n in this.tables) { 
         newStatistics.tables[n+'_transposed'] = this.tables[n].transpose(n) 
      }
      return newStatistics
   }
}
export default Statistics;