ai-patterns
Version:
Production-ready TypeScript patterns to build solid and robust AI applications. Retry logic, circuit breakers, rate limiting, human-in-the-loop escalation, prompt versioning, response validation, context window management, and more—all with complete type
192 lines • 7.28 kB
JavaScript
;
/**
* A/B Testing Pattern
*
* Allows testing multiple variants simultaneously and measuring their performance
* to continuously optimize AI applications.
*
* @example
* ```typescript
* const result = await abTest({
* variants: [
* {
* name: 'Simple',
* weight: 0.33,
* execute: async () => generateText({ prompt: 'Explain quantum computing' })
* },
* {
* name: 'With Context',
* weight: 0.33,
* execute: async () => generateText({
* prompt: 'Explain quantum computing to a software developer'
* })
* }
* ],
* userId: 'user-123',
* onVariantSelected: (variant, result) => {
* analytics.track('variant_selected', { variant: variant.name });
* }
* });
* ```
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.InMemoryAssignmentStorage = void 0;
exports.abTest = abTest;
const ab_test_1 = require("../types/ab-test");
const common_1 = require("../types/common");
const errors_1 = require("../types/errors");
const storage_1 = require("../common/storage");
/**
* Simple in-memory storage for sticky assignments using GlobalStorage
*/
class InMemoryAssignmentStorage {
constructor() {
this.namespace = storage_1.StorageNamespace.AB_TEST;
this.storage = storage_1.GlobalStorage.getInstance();
}
async get(userId, experimentId) {
const key = `${experimentId}:${userId}`;
const value = await this.storage.get(this.namespace, key);
return value ?? null;
}
async set(userId, experimentId, variantName) {
const key = `${experimentId}:${userId}`;
await this.storage.set(this.namespace, key, variantName);
}
}
exports.InMemoryAssignmentStorage = InMemoryAssignmentStorage;
/**
* Default in-memory storage instance (lazy initialization)
*/
let defaultStorage = null;
function getDefaultStorage() {
if (!defaultStorage) {
defaultStorage = new InMemoryAssignmentStorage();
}
return defaultStorage;
}
/**
* Hash function for consistent variant assignment
*/
function hashCode(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash = hash & hash; // Convert to 32-bit integer
}
return Math.abs(hash);
}
/**
* Select a variant based on weights
*/
function selectVariantByWeight(variants, random = Math.random()) {
// Normalize weights to sum to 1.0
const totalWeight = variants.reduce((sum, v) => sum + v.weight, 0);
const normalizedVariants = variants.map((v) => ({
...v,
weight: v.weight / totalWeight,
}));
let cumulative = 0;
for (const variant of normalizedVariants) {
cumulative += variant.weight;
if (random <= cumulative) {
return variant;
}
}
// Fallback to last variant (should not happen with proper weights)
return variants[variants.length - 1];
}
/**
* Select a variant for a specific user (sticky assignment)
*/
function selectVariantForUser(variants, userId, experimentId) {
const key = `${experimentId}:${userId}`;
const hash = hashCode(key);
const random = (hash % 10000) / 10000; // Normalize to 0-1
return selectVariantByWeight(variants, random);
}
/**
* Execute an A/B test with the given configuration
*/
async function abTest(config) {
const { variants, userId, experimentId = "default", strategy = ab_test_1.VariantAssignmentStrategy.WEIGHTED, storage = getDefaultStorage(), onVariantSelected, onSuccess, onError, logger = common_1.defaultLogger, } = config;
// Validate variants
if (!variants || variants.length === 0) {
throw new errors_1.PatternError("At least one variant is required for A/B testing", errors_1.ErrorCode.NO_VARIANTS);
}
// Validate weights
const totalWeight = variants.reduce((sum, v) => sum + v.weight, 0);
if (Math.abs(totalWeight - 1.0) > 0.001) {
logger.warn(`Variant weights sum to ${totalWeight}, expected 1.0. Weights will be normalized.`);
}
// Select variant based on strategy
let selectedVariant;
if (userId && strategy === ab_test_1.VariantAssignmentStrategy.STICKY) {
// Check for existing sticky assignment
const assignedVariantName = await storage.get(userId, experimentId);
if (assignedVariantName) {
const found = variants.find((v) => v.name === assignedVariantName);
if (found) {
selectedVariant = found;
logger.debug(`Using sticky assignment: variant "${selectedVariant.name}" for user ${userId}`);
}
else {
// Variant no longer exists, select new one
selectedVariant = selectVariantForUser(variants, userId, experimentId);
await storage.set(userId, experimentId, selectedVariant.name);
logger.debug(`Sticky variant not found, selected new variant "${selectedVariant.name}" for user ${userId}`);
}
}
else {
// No existing assignment
selectedVariant = selectVariantForUser(variants, userId, experimentId);
await storage.set(userId, experimentId, selectedVariant.name);
logger.debug(`New sticky assignment: variant "${selectedVariant.name}" for user ${userId}`);
}
}
else if (userId && strategy === ab_test_1.VariantAssignmentStrategy.WEIGHTED) {
// Weighted selection with user consistency
selectedVariant = selectVariantForUser(variants, userId, experimentId);
logger.debug(`Weighted selection: variant "${selectedVariant.name}" for user ${userId}`);
}
else {
// Random selection
selectedVariant = selectVariantByWeight(variants);
logger.debug(`Random selection: variant "${selectedVariant.name}"`);
}
const timestamp = Date.now();
try {
// Execute the selected variant
const result = await selectedVariant.execute();
// Call onVariantSelected callback
if (onVariantSelected) {
await onVariantSelected(selectedVariant, result);
}
// Call onSuccess callback
if (onSuccess) {
await onSuccess(selectedVariant, result);
}
logger.info(`A/B test completed successfully with variant "${selectedVariant.name}"`);
return {
variant: selectedVariant,
value: result,
timestamp,
userId,
experimentId,
};
}
catch (error) {
const variantError = error instanceof Error ? error : new Error(String(error));
logger.error(`A/B test failed with variant "${selectedVariant.name}"`, {
error: variantError.message,
});
// Call onError callback
if (onError) {
await onError(selectedVariant, variantError);
}
// Wrap in PatternError for consistency
throw new errors_1.PatternError(`Variant "${selectedVariant.name}" execution failed: ${variantError.message}`, errors_1.ErrorCode.VARIANT_EXECUTION_FAILED, variantError, { variantName: selectedVariant.name, experimentId, userId, strategy });
}
}
//# sourceMappingURL=ab-test.js.map