react-alpha-beta/lib/Experiment.js

Simple declarative A/B testing component for React.

'use strict';

Object.defineProperty(exports, "__esModule", {
  value: true
});

var _stringify = require('babel-runtime/core-js/json/stringify');

var _stringify2 = _interopRequireDefault(_stringify);

exports.postExperimentData = postExperimentData;
exports.getPooledVariance = getPooledVariance;
exports.getPooledStandardDeviation = getPooledStandardDeviation;
exports.getUnpooledVariance = getUnpooledVariance;
exports.assumeNormalDistribution = assumeNormalDistribution;
exports.computeStats = computeStats;
exports.getExperimentData = getExperimentData;

var _isomorphicFetch = require('isomorphic-fetch');

var _isomorphicFetch2 = _interopRequireDefault(_isomorphicFetch);

var _config = require('./config');

var _config2 = _interopRequireDefault(_config);

var _zTable = require('./zTable');

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function postExperimentData(experimentId, variant) {
  var success = arguments.length <= 2 || arguments[2] === undefined ? null : arguments[2];
  var metaId = arguments.length <= 3 || arguments[3] === undefined ? null : arguments[3];

  return (0, _isomorphicFetch2.default)(_config2.default.endPoint + '/' + experimentId + '/', {
    method: 'POST',
    credentials: 'same-origin',
    headers: {
      'Accept': 'application/json',
      'Content-Type': 'application/json'
    },
    body: (0, _stringify2.default)({
      experimentId: experimentId,
      metaId: metaId,
      success: success,
      userCohortValue: JSON.parse(global.localStorage.getItem('alphaBetaMap'))[experimentId],
      variant: variant
    })
  });
}

function getPooledVariance(trialsA, trialsB, successA, successB) {
  var successPooled = (successA + successB) / (trialsA + trialsB);
  return successPooled * (1 - successPooled) * (1 / trialsA + 1 / trialsB);
}

/**
 * Function to return the pooled estimate of the common standard deviation (Sp).
 */
function getPooledStandardDeviation(trialsA, trialsB, successA, successB) {
  return Math.sqrt(getPooledVariance(trialsA, trialsB, successA, successB));
}

function getUnpooledVariance(trialsA, trialsB, varianceA, varianceB) {
  return Math.sqrt(varianceA / trialsA + varianceB / trialsB);
}

function assumeNormalDistribution(trials, probabilityMean) {
  // heuristic: when the # of trials * the mean probability of success is > 5,
  // it's safe to assume the normal distrabution can be used (instead of the
  // binomieal distrabution)
  return trials * probabilityMean > 5;
}

function testDetails(probabilityMeanDifference, marginOfError, confidenceInterval) {
  // a function that turns the numeric data from a test into a short, human readable
  // description of the findings
  var differenceFloor = probabilityMeanDifference - marginOfError;
  var differenceCeiling = probabilityMeanDifference + marginOfError;

  var details = void 0;
  var range = 'We are ' + Math.round(confidenceInterval * 100, -2) + '% confident the true difference is between ' + Math.round(differenceFloor * 100, -2) + '% and ' + Math.round(differenceCeiling * 100, -2) + '%.';
  var recommendation = void 0;
  if (probabilityMeanDifference > 0) {
    details = 'Our best estimate is that the absolute rate of success is ' + Math.round(Math.abs(probabilityMeanDifference) * 100, -2) + '% higher with variant B';
  } else {
    details = 'Our best estimate is that the absolute rate of success is ' + Math.round(Math.abs(probabilityMeanDifference) * 100, -2) + '% lower with variant B';
  }

  if (differenceFloor * differenceCeiling > 0) {
    details = details + ', and this result is statistically significant';
    if (probabilityMeanDifference > 0) {
      recommendation = 'It looks like a safe bet to go with variant B.';
    } else {
      recommendation = 'Given this information, you should probably stick with variant A.';
    }
  } else {
    details = details + ', but this result is not statistically significant';
    recommendation = 'You don\'t yet have enough information to make a confident decision, so you should keep running this experiment.';
  }
  return details + ' (' + range + '). ' + recommendation;
}

function computeStats(json) {
  var variantATrialCount = json.variantA.trialCount;
  var variantBTrialCount = json.variantB.trialCount;
  var variantASuccessCount = json.variantA.successCount;
  var variantBSuccessCount = json.variantB.successCount;
  var confidenceInterval = json.confidenceInterval;

  var probabilityMeanA = variantASuccessCount / variantATrialCount;
  var probabilityMeanB = variantBSuccessCount / variantBTrialCount;
  var probabilityMeanDifference = probabilityMeanB - probabilityMeanA;
  var probabilityVarianceA = variantATrialCount * probabilityMeanA * (1 - probabilityMeanA);
  var probabilityVarianceB = variantBTrialCount * probabilityMeanB * (1 - probabilityMeanB);
  var varianceRatio = probabilityVarianceA / probabilityVarianceB;
  var probabilityVariancePooled = getPooledStandardDeviation(variantATrialCount, variantBTrialCount, variantASuccessCount, variantBSuccessCount);
  var probabilityVarianceUnpooled = getUnpooledVariance(variantATrialCount, variantBTrialCount, probabilityVarianceA, probabilityVarianceB);
  var zScore = (0, _zTable.zScoreByConfidenceInterval)(confidenceInterval);

  var assumeNormalDistributionA = assumeNormalDistribution(variantATrialCount, probabilityMeanA);
  var assumeNormalDistributionB = assumeNormalDistribution(variantBTrialCount, probabilityMeanB);

  if (assumeNormalDistributionA === false || assumeNormalDistributionB === false) {
    return {
      statisticalSignificance: false,
      details: 'You do not have enough sample data for one or both of your variants to make any assertions.'
    };
  }

  var marginOfError = void 0;
  if (varianceRatio <= 0.5 || varianceRatio >= 2) {
    // heuristic: when one variance is more than double the other, we cannot use the
    // pooled estimate of the common standard deviation.
    marginOfError = zScore * probabilityVarianceUnpooled * Math.sqrt(1 / variantATrialCount + 1 / variantBTrialCount);
  } else {
    marginOfError = zScore * probabilityVariancePooled * Math.sqrt(1 / variantATrialCount + 1 / variantBTrialCount);
  }

  var differenceFloor = probabilityMeanDifference - marginOfError;
  var differenceCeiling = probabilityMeanDifference + marginOfError;
  var statisticalSignificance = differenceFloor * differenceCeiling > 0;

  var result = {
    statisticalSignificance: statisticalSignificance,
    meanDifferenceValue: probabilityMeanDifference,
    marginOfError: marginOfError,
    details: testDetails(probabilityMeanDifference, marginOfError, confidenceInterval)
  };
  return result;
}

function getExperimentData(experimentId) {
  return (0, _isomorphicFetch2.default)(_config2.default.endPoint + '/' + experimentId + '/', {
    method: 'GET',
    credentials: 'same-origin',
    headers: {
      'Accept': 'application/json',
      'Content-Type': 'application/json'
    }
  }).then(function (response) {
    return response.json();
  }).then(function (json) {
    return computeStats(json);
  });
}