@varia-bly/variably-sdk
Version:
Official JavaScript/TypeScript SDK for Variably feature flags, experimentation, LLM experiments with React hooks, and real-time dynamic configurations
233 lines • 10.5 kB
JavaScript
/**
* Statistical analysis utilities for A/B testing and experimentation
* Implements advanced analytics features from the metrics collection documentation
*/
export class StatisticalAnalysis {
/**
* Calculate statistical significance for A/B test
*/
static calculateABTestSignificance(treatmentConversions, treatmentTotal, controlConversions, controlTotal, confidenceLevel = 0.95) {
const treatmentRate = treatmentConversions / treatmentTotal;
const controlRate = controlConversions / controlTotal;
const conversionLift = treatmentRate - controlRate;
const conversionLiftPercent = ((treatmentRate - controlRate) / controlRate) * 100;
// Calculate standard error for two-proportion z-test
const pooledRate = (treatmentConversions + controlConversions) / (treatmentTotal + controlTotal);
const standardError = Math.sqrt(pooledRate * (1 - pooledRate) * (1 / treatmentTotal + 1 / controlTotal));
// Calculate z-score
const zScore = Math.abs(conversionLift) / standardError;
// Calculate p-value (two-tailed test)
const pValue = 2 * (1 - this.normalCDF(zScore));
const zCritical = confidenceLevel === 0.99 ?
this.Z_SCORE_99_CONFIDENCE :
this.Z_SCORE_95_CONFIDENCE;
const isSignificant = zScore > zCritical;
// Calculate confidence interval for difference in proportions
const ciStandardError = Math.sqrt((treatmentRate * (1 - treatmentRate)) / treatmentTotal +
(controlRate * (1 - controlRate)) / controlTotal);
const marginOfError = zCritical * ciStandardError;
const confidenceInterval = [
conversionLift - marginOfError,
conversionLift + marginOfError
];
// Calculate minimum detectable effect (MDE) at 80% power
const mde = this.calculateMDE(controlRate, treatmentTotal, controlTotal, 0.8, confidenceLevel);
// Calculate statistical power
const power = this.calculatePower(controlRate, treatmentRate, treatmentTotal, controlTotal, confidenceLevel);
return {
treatmentGroup: 'treatment',
controlGroup: 'control',
treatmentConversion: treatmentRate,
controlConversion: controlRate,
treatmentSampleSize: treatmentTotal,
controlSampleSize: controlTotal,
conversionLift,
conversionLiftPercent,
pValue,
statisticalSignificance: isSignificant,
confidenceLevel,
confidenceInterval,
minimumDetectableEffect: mde,
power
};
}
/**
* Calculate required sample size for A/B test
*/
static calculateRequiredSampleSize(baselineConversion, minimumDetectableEffect, power = 0.8, confidenceLevel = 0.95) {
const zAlpha = confidenceLevel === 0.99 ? this.Z_SCORE_99_CONFIDENCE : this.Z_SCORE_95_CONFIDENCE;
const zBeta = this.getZScoreForPower(power);
const p1 = baselineConversion;
const p2 = baselineConversion + minimumDetectableEffect;
const pooledP = (p1 + p2) / 2;
const pooledVariance = pooledP * (1 - pooledP);
const individualVariance = p1 * (1 - p1) + p2 * (1 - p2);
const numerator = Math.pow(zAlpha * Math.sqrt(2 * pooledVariance) + zBeta * Math.sqrt(individualVariance), 2);
const denominator = Math.pow(p2 - p1, 2);
return Math.ceil(numerator / denominator);
}
/**
* Perform cohort analysis for user retention
*/
static analyzeCohort(cohortUsers, userEvents, retentionEvent = 'session_start', periods = ['day_1', 'day_7', 'day_30', 'day_90']) {
const cohortStartDate = new Date(Math.min(...userEvents
.filter(e => cohortUsers.includes(e.userId))
.map(e => e.timestamp.getTime())));
const retentionRates = periods.map(period => {
const daysOffset = this.parsePeriodToDays(period);
const targetDate = new Date(cohortStartDate.getTime() + daysOffset * 24 * 60 * 60 * 1000);
const retainedUsers = cohortUsers.filter(userId => {
return userEvents.some(event => event.userId === userId &&
event.eventName === retentionEvent &&
event.timestamp >= targetDate &&
event.timestamp < new Date(targetDate.getTime() + 24 * 60 * 60 * 1000));
}).length;
return {
period,
retainedUsers,
retentionRate: retainedUsers / cohortUsers.length
};
});
// Calculate churn rate (inverse of day_30 retention)
const day30Retention = retentionRates.find(r => r.period === 'day_30');
const churnRate = day30Retention ? 1 - day30Retention.retentionRate : 0;
return {
cohortName: `Cohort_${cohortStartDate.toISOString().split('T')[0]}`,
cohortSize: cohortUsers.length,
retentionRates,
churnRate
};
}
/**
* Analyze conversion funnel
*/
static analyzeFunnel(funnelSteps, userEvents, timeWindow = 24 * 60 * 60 * 1000 // 24 hours in milliseconds
) {
// Get all users who completed the first step
const firstStepUsers = [
...new Set(userEvents
.filter(e => e.eventName === funnelSteps[0])
.map(e => e.userId))
];
const totalUsers = firstStepUsers.length;
const stepResults = [];
let previousStepUsers = firstStepUsers;
for (let i = 0; i < funnelSteps.length; i++) {
const stepName = funnelSteps[i];
if (i === 0) {
// First step: all users who performed this event
stepResults.push({
stepName,
eventName: stepName,
userCount: totalUsers,
conversionRate: 1.0,
dropoffRate: 0.0
});
}
else {
// Subsequent steps: users who performed this event within time window after previous step
const stepUsers = previousStepUsers.filter(userId => {
const previousStepEvent = userEvents
.filter(e => e.userId === userId && e.eventName === funnelSteps[i - 1])
.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime())[0];
if (!previousStepEvent)
return false;
const nextStepEvent = userEvents.find(e => e.userId === userId &&
e.eventName === stepName &&
e.timestamp.getTime() > previousStepEvent.timestamp.getTime() &&
e.timestamp.getTime() <= previousStepEvent.timestamp.getTime() + timeWindow);
return !!nextStepEvent;
});
const conversionRate = stepUsers.length / previousStepUsers.length;
const dropoffRate = 1 - conversionRate;
stepResults.push({
stepName,
eventName: stepName,
userCount: stepUsers.length,
conversionRate,
dropoffRate
});
previousStepUsers = stepUsers;
}
}
// Find biggest dropoff
let biggestDropoff = {
fromStep: '',
toStep: '',
dropoffRate: 0
};
for (let i = 1; i < stepResults.length; i++) {
if (stepResults[i].dropoffRate > biggestDropoff.dropoffRate) {
biggestDropoff = {
fromStep: stepResults[i - 1].stepName,
toStep: stepResults[i].stepName,
dropoffRate: stepResults[i].dropoffRate
};
}
}
const overallConversionRate = stepResults[stepResults.length - 1].userCount / totalUsers;
return {
funnelName: `Funnel_${funnelSteps.join('_')}`,
totalUsers,
overallConversionRate,
steps: stepResults,
biggestDropoff
};
}
/**
* Normal cumulative distribution function approximation
*/
static normalCDF(x) {
// Abramowitz and Stegun approximation
const t = 1 / (1 + 0.2316419 * Math.abs(x));
const d = 0.3989423 * Math.exp(-x * x / 2);
const prob = d * t * (0.3193815 + t * (-0.3565638 + t * (1.7814779 + t * (-1.8212560 + t * 1.3302744))));
return x > 0 ? 1 - prob : prob;
}
/**
* Calculate Minimum Detectable Effect
*/
static calculateMDE(baselineRate, treatmentSize, controlSize, power, confidenceLevel) {
const zAlpha = confidenceLevel === 0.99 ? this.Z_SCORE_99_CONFIDENCE : this.Z_SCORE_95_CONFIDENCE;
const zBeta = this.getZScoreForPower(power);
const harmonicMean = 2 / (1 / treatmentSize + 1 / controlSize);
const variance = baselineRate * (1 - baselineRate);
return (zAlpha + zBeta) * Math.sqrt(2 * variance / harmonicMean);
}
/**
* Calculate statistical power
*/
static calculatePower(controlRate, treatmentRate, treatmentSize, controlSize, confidenceLevel) {
const zAlpha = confidenceLevel === 0.99 ? this.Z_SCORE_99_CONFIDENCE : this.Z_SCORE_95_CONFIDENCE;
const effect = Math.abs(treatmentRate - controlRate);
const pooledVariance = ((controlRate * (1 - controlRate)) / controlSize) +
((treatmentRate * (1 - treatmentRate)) / treatmentSize);
const standardError = Math.sqrt(pooledVariance);
const zBeta = (effect / standardError) - zAlpha;
return this.normalCDF(zBeta);
}
/**
* Get Z-score for given statistical power
*/
static getZScoreForPower(power) {
// Common power levels and their Z-scores
const powerMap = {
0.5: 0,
0.8: 0.842,
0.9: 1.282,
0.95: 1.645,
0.99: 2.326
};
return powerMap[power] || 0.842; // Default to 80% power
}
/**
* Parse period string to days
*/
static parsePeriodToDays(period) {
const match = period.match(/day_(\d+)/);
return match ? parseInt(match[1], 10) : 1;
}
}
StatisticalAnalysis.Z_SCORE_95_CONFIDENCE = 1.96;
StatisticalAnalysis.Z_SCORE_99_CONFIDENCE = 2.576;
//# sourceMappingURL=statistical-analysis.js.map