hypertune/src/shared/helpers/calculateFrequentistMetrics.ts

[Hypertune](https://www.hypertune.com/) is the most flexible platform for feature flags, A/B testing, analytics and app configuration. Built with full end-to-end type-safety, Git-style version control and local, synchronous, in-memory flag evaluation. Opt

function calculateZScore(
  conversionRateA: number,
  conversionRateB: number,
  standardError: number
): number {
  return standardError
    ? (conversionRateB / conversionRateA - 1) / standardError
    : 0;
}

export type FrequentistMetricsResult = {
  upLift: number;
  lowerBound: number;
  upperBound: number;
  isSignificant: boolean;
};

function getZValueThreshold(confidence: number, oneSided: boolean): number {
  switch (oneSided) {
    case false:
      switch (confidence) {
        case 0.9:
          return 1.645;
        case 0.95:
          return 1.96;
        case 0.99:
          return 2.576;
        default:
          throw Error("Invalid confidence value");
      }
    default:
      switch (confidence) {
        case 0.9:
          return 1.28;
        case 0.95:
          return 1.645;
        case 0.99:
          return 2.33;
        default:
          throw Error("Invalid confidence value");
      }
  }
}

// eslint-disable-next-line max-params
export default function calculateFrequentistMetrics(
  successes: number[],
  visitors: number[],
  controlIndex: number,
  confidence: number,
  oneSided: boolean
): { [index: number]: FrequentistMetricsResult } | null {
  const armMetrics: { [index: number]: FrequentistMetricsResult } = {};
  if (
    controlIndex < 0 ||
    controlIndex >= successes.length ||
    controlIndex >= visitors.length
  ) {
    return null;
  }
  successes.forEach((successB, i) => {
    const successA = successes[controlIndex];
    const visitorB = visitors[i];
    const visitorA = visitors[controlIndex];
    const conversionRateA = successA / visitorA;
    const conversionRateB = successB / visitorB;

    const standardErrorUplift =
      Math.sqrt(
        (conversionRateA * (1 - conversionRateA)) / visitorA +
          (conversionRateB * (1 - conversionRateB)) / visitorB
      ) / conversionRateA;

    const z = calculateZScore(
      conversionRateA,
      conversionRateB,
      standardErrorUplift
    );
    const upLift = (conversionRateB - conversionRateA) / conversionRateA;
    const isSignificant = oneSided
      ? z > getZValueThreshold(confidence, oneSided)
      : Math.abs(z) > getZValueThreshold(confidence, oneSided);

    const marginOfError =
      getZValueThreshold(confidence, oneSided) * standardErrorUplift;
    const lowerBound = upLift - marginOfError;
    const upperBound = upLift + marginOfError;

    armMetrics[i] = {
      upLift,
      lowerBound,
      upperBound,
      isSignificant,
    };
  });

  return armMetrics;
}