hypertune/src/shared/helpers/calculateFrequentistMetrics.test.ts

[Hypertune](https://www.hypertune.com/) is the most flexible platform for feature flags, A/B testing, analytics and app configuration. Built with full end-to-end type-safety, Git-style version control and local, synchronous, in-memory flag evaluation. Opt

import { test, expect, describe } from "vitest";
import calculateFrequentistMetrics from "./calculateFrequentistMetrics";

describe("calculateBayesianProbability", () => {
  const cases = [
    {
      name: "Test positive conversion: 99% confidence",
      successes: [1600, 1680],
      visitors: [80000, 80000],
      confidence: 0.99,
      expected: [
        { isSignificant: false, upLift: "0.00", lowerBoundLessThanZero: true },
        {
          isSignificant: false,
          upLift: "0.0500",
          lowerBoundLessThanZero: true,
        },
      ],
    },
    {
      name: "Test positive conversion, 95% confidence",
      successes: [1600, 1680],
      visitors: [80000, 80000],
      confidence: 0.95,
      expected: [
        {
          isSignificant: false,
          upLift: "0.00",
          lowerBoundLessThanZero: true,
        },
        {
          isSignificant: false,
          upLift: "0.0500",
          lowerBoundLessThanZero: true,
        },
      ],
    },
    {
      name: "Test positive conversion: 90% confidence",
      successes: [1600, 1680],
      visitors: [80000, 80000],
      confidence: 0.9,
      expected: [
        {
          isSignificant: false,
          upLift: "0.00",
          lowerBoundLessThanZero: true,
        },
        {
          isSignificant: true,
          upLift: "0.0500",
          lowerBoundLessThanZero: false,
        },
      ],
    },
    {
      name: "Test negative conversion: 99% confidence",
      successes: [1600, 1590],
      visitors: [80000, 80000],
      confidence: 0.99,
      expected: [
        {
          isSignificant: false,
          upLift: "0.00",
          lowerBoundLessThanZero: true,
        },
        {
          isSignificant: false,
          upLift: "-0.00625",
          lowerBoundLessThanZero: true,
        },
      ],
    },
    {
      name: "Test negative conversion, 95% confidence",
      successes: [1600, 1590],
      visitors: [80000, 80000],
      confidence: 0.95,
      expected: [
        {
          isSignificant: false,
          upLift: "0.00",
          lowerBoundLessThanZero: true,
        },
        {
          isSignificant: false,
          upLift: "-0.00625",
          lowerBoundLessThanZero: true,
        },
      ],
    },
    {
      name: "Test negative conversion: 90% confidence",
      successes: [1600, 1590],
      visitors: [80000, 80000],
      confidence: 0.9,
      expected: [
        {
          isSignificant: false,
          upLift: "0.00",
          lowerBoundLessThanZero: true,
        },
        {
          isSignificant: false,
          upLift: "-0.00625",
          lowerBoundLessThanZero: true,
        },
      ],
    },
  ];

  test.each(cases)("$name", ({ successes, visitors, confidence, expected }) => {
    expect(
      Object.entries(
        calculateFrequentistMetrics(successes, visitors, 0, confidence)
      ).forEach(
        ([index, { upLift, isSignificant, lowerBound, upperBound }]) => {
          expect(isSignificant).toEqual(expected[index].isSignificant);
          expect(upLift.toPrecision(3)).toEqual(expected[index].upLift);
          if (expected[index].lowerBoundLessThanZero) {
            expect(lowerBound).toBeLessThanOrEqual(0);
            return;
          }

          expect(lowerBound).toBeGreaterThan(0);
          expect(upLift).toBeGreaterThanOrEqual(lowerBound);
          expect(upLift).toBeLessThanOrEqual(upperBound);
        }
      )
    );
  });
});