@callmedayz/ai-prompt-toolkit
Version:
Professional AI prompt engineering toolkit with advanced template features, real-time dashboards, conditional logic, template inheritance, live monitoring, OpenRouter integration, and 310+ model support
448 lines • 16.6 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.PromptVersionManager = void 0;
exports.createQuickABTest = createQuickABTest;
const prompt_template_1 = require("./prompt-template");
const openrouter_completion_1 = require("./openrouter-completion");
const utils_1 = require("./utils");
const openrouter_types_1 = require("./openrouter-types");
/**
* Prompt Versioning and A/B Testing System
*/
class PromptVersionManager {
constructor(client) {
this.versions = new Map();
this.activeTests = new Map();
this.testExecutions = new Map();
this.client = client;
if (client) {
this.completion = new openrouter_completion_1.OpenRouterCompletion(client);
}
}
/**
* Create a new prompt version
*/
createVersion(name, template, variables = {}, options = {}) {
const id = this.generateId();
const version = this.generateVersionNumber(options.parentVersion);
const promptVersion = {
id,
version,
name,
description: options.description,
template,
variables,
metadata: {
createdAt: new Date(),
createdBy: options.createdBy,
tags: options.tags || [],
parentVersion: options.parentVersion,
isActive: true
}
};
this.versions.set(id, promptVersion);
return promptVersion;
}
/**
* Get a prompt version by ID
*/
getVersion(id) {
return this.versions.get(id);
}
/**
* List all versions of a prompt by name
*/
getVersionsByName(name) {
return Array.from(this.versions.values())
.filter(v => v.name === name)
.sort((a, b) => b.metadata.createdAt.getTime() - a.metadata.createdAt.getTime());
}
/**
* Get the latest active version of a prompt
*/
getLatestVersion(name) {
const versions = this.getVersionsByName(name)
.filter(v => v.metadata.isActive);
return versions.length > 0 ? versions[0] : undefined;
}
/**
* Update a prompt version (creates a new version)
*/
updateVersion(parentId, updates) {
const parent = this.getVersion(parentId);
if (!parent) {
throw new Error(`Parent version ${parentId} not found`);
}
return this.createVersion(parent.name, updates.template || parent.template, { ...parent.variables, ...updates.variables }, {
description: updates.description || parent.description,
tags: updates.tags || parent.metadata.tags,
parentVersion: parentId
});
}
/**
* Deactivate a prompt version
*/
deactivateVersion(id) {
const version = this.getVersion(id);
if (version) {
version.metadata.isActive = false;
}
}
generateId() {
return `pv_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
generateVersionNumber(parentVersion) {
if (!parentVersion) {
return '1.0.0';
}
const parent = this.getVersion(parentVersion);
if (!parent) {
return '1.0.0';
}
const [major, minor, patch] = parent.version.split('.').map(Number);
return `${major}.${minor}.${patch + 1}`;
}
/**
* Start an A/B test with multiple prompt versions
*/
async startABTest(config) {
// Validate config
this.validateABTestConfig(config);
const testId = this.generateId();
const testResult = {
testId,
config,
variants: config.variants.map(version => ({
version,
metrics: {
totalTests: 0,
successRate: 0,
averageResponseTime: 0,
averageTokenUsage: 0,
averageCost: 0,
lastTested: new Date()
},
testCount: 0
})),
startTime: new Date(),
status: 'running',
confidence: 0,
summary: 'Test started'
};
this.activeTests.set(testId, testResult);
this.testExecutions.set(testId, []);
return testResult;
}
/**
* Execute a test for a specific variant in an A/B test
*/
async executeTest(testId, input, model = openrouter_types_1.DEFAULT_FREE_MODEL, config = {}) {
const test = this.activeTests.get(testId);
if (!test || test.status !== 'running') {
throw new Error(`Test ${testId} is not running`);
}
if (!this.completion) {
throw new Error('OpenRouter client not configured for testing');
}
// Select variant based on traffic split
const variant = this.selectVariant(test);
const promptTemplate = new prompt_template_1.PromptTemplate({
template: variant.version.template,
variables: variant.version.variables
});
const prompt = promptTemplate.render({ input });
const startTime = Date.now();
try {
const result = await this.completion.complete(prompt, {
model,
...config
});
const endTime = Date.now();
const responseTime = endTime - startTime;
// Estimate token usage and cost
const tokenInfo = (0, utils_1.estimateTokens)(prompt + result.text, model);
const execution = {
id: this.generateId(),
promptVersionId: variant.version.id,
input,
output: result.text,
responseTime,
tokenUsage: tokenInfo.tokens,
cost: tokenInfo.estimatedCost || 0,
success: true,
timestamp: new Date()
};
// Record the execution
this.recordTestExecution(testId, execution);
return execution;
}
catch (error) {
const endTime = Date.now();
const responseTime = endTime - startTime;
const execution = {
id: this.generateId(),
promptVersionId: variant.version.id,
input,
output: '',
responseTime,
tokenUsage: 0,
cost: 0,
success: false,
timestamp: new Date(),
metadata: { error: error instanceof Error ? error.message : String(error) }
};
this.recordTestExecution(testId, execution);
throw error;
}
}
/**
* Get the current status of an A/B test
*/
getTestStatus(testId) {
return this.activeTests.get(testId);
}
/**
* Stop an A/B test and analyze results
*/
stopABTest(testId) {
const test = this.activeTests.get(testId);
if (!test) {
throw new Error(`Test ${testId} not found`);
}
test.status = 'stopped';
test.endTime = new Date();
// Analyze results and determine winner
this.analyzeTestResults(test);
return test;
}
/**
* Complete an A/B test and analyze results
*/
completeABTest(testId) {
const test = this.activeTests.get(testId);
if (!test) {
throw new Error(`Test ${testId} not found`);
}
test.status = 'completed';
test.endTime = new Date();
// Analyze results and determine winner
this.analyzeTestResults(test);
return test;
}
/**
* Get test executions for a specific test
*/
getTestExecutions(testId) {
return this.testExecutions.get(testId) || [];
}
/**
* Export prompt versions to JSON
*/
exportVersions() {
const data = {
versions: Array.from(this.versions.values()),
exportedAt: new Date().toISOString()
};
return JSON.stringify(data, null, 2);
}
/**
* Import prompt versions from JSON
*/
importVersions(jsonData) {
try {
const data = JSON.parse(jsonData);
if (data.versions && Array.isArray(data.versions)) {
data.versions.forEach((version) => {
// Convert date strings back to Date objects
version.metadata.createdAt = new Date(version.metadata.createdAt);
if (version.performance?.lastTested) {
version.performance.lastTested = new Date(version.performance.lastTested);
}
this.versions.set(version.id, version);
});
}
}
catch (error) {
throw new Error(`Failed to import versions: ${error instanceof Error ? error.message : String(error)}`);
}
}
validateABTestConfig(config) {
if (config.variants.length < 2) {
throw new Error('A/B test requires at least 2 variants');
}
if (config.trafficSplit.length !== config.variants.length) {
throw new Error('Traffic split array must match number of variants');
}
const totalSplit = config.trafficSplit.reduce((sum, split) => sum + split, 0);
if (Math.abs(totalSplit - 100) > 0.01) {
throw new Error('Traffic split must sum to 100%');
}
config.trafficSplit.forEach(split => {
if (split < 0 || split > 100) {
throw new Error('Traffic split values must be between 0 and 100');
}
});
}
selectVariant(test) {
const random = Math.random() * 100;
let cumulative = 0;
for (let i = 0; i < test.config.trafficSplit.length; i++) {
cumulative += test.config.trafficSplit[i];
if (random <= cumulative) {
return test.variants[i];
}
}
// Fallback to first variant
return test.variants[0];
}
recordTestExecution(testId, execution) {
const executions = this.testExecutions.get(testId) || [];
executions.push(execution);
this.testExecutions.set(testId, executions);
// Update test metrics
const test = this.activeTests.get(testId);
if (test) {
const variant = test.variants.find(v => v.version.id === execution.promptVersionId);
if (variant) {
variant.testCount++;
this.updateVariantMetrics(variant, executions.filter(e => e.promptVersionId === execution.promptVersionId));
}
}
}
updateVariantMetrics(variant, executions) {
const successfulExecutions = executions.filter(e => e.success);
variant.metrics.totalTests = executions.length;
variant.metrics.successRate = executions.length > 0 ? (successfulExecutions.length / executions.length) * 100 : 0;
variant.metrics.averageResponseTime = executions.length > 0
? executions.reduce((sum, e) => sum + e.responseTime, 0) / executions.length
: 0;
variant.metrics.averageTokenUsage = executions.length > 0
? executions.reduce((sum, e) => sum + e.tokenUsage, 0) / executions.length
: 0;
variant.metrics.averageCost = executions.length > 0
? executions.reduce((sum, e) => sum + e.cost, 0) / executions.length
: 0;
variant.metrics.lastTested = new Date();
// Update the version's performance metrics
variant.version.performance = { ...variant.metrics };
}
analyzeTestResults(test) {
if (test.variants.length === 0) {
test.summary = 'No variants to analyze';
return;
}
// Determine winner based on success criteria
let winner;
let bestScore = -Infinity;
for (const variant of test.variants) {
let score = 0;
let criteriaMetCount = 0;
for (const criteria of test.config.successCriteria) {
const metricValue = this.getMetricValue(variant.metrics, criteria.metric);
const meetsCriteria = this.evaluateCriteria(metricValue, criteria.target, criteria.operator);
if (meetsCriteria) {
criteriaMetCount++;
// Weight the score based on the metric type
switch (criteria.metric) {
case 'success_rate':
score += metricValue * 2; // Higher weight for success rate
break;
case 'response_time':
score += criteria.operator === 'less_than' ? (1000 - metricValue) : metricValue;
break;
case 'cost':
score += criteria.operator === 'less_than' ? (1 - metricValue) * 1000 : metricValue;
break;
default:
score += metricValue;
}
}
}
// Bonus for meeting more criteria
score += criteriaMetCount * 100;
if (score > bestScore && variant.testCount > 0) {
bestScore = score;
winner = variant;
}
}
if (winner) {
winner.isWinner = true;
test.winner = winner.version;
test.confidence = this.calculateConfidence(test);
test.summary = `Winner: ${winner.version.name} (v${winner.version.version}) with ${winner.testCount} tests and ${winner.metrics.successRate.toFixed(1)}% success rate`;
}
else {
test.summary = 'No clear winner based on success criteria';
test.confidence = 0;
}
}
getMetricValue(metrics, metric) {
switch (metric) {
case 'response_time':
return metrics.averageResponseTime;
case 'token_usage':
return metrics.averageTokenUsage;
case 'cost':
return metrics.averageCost;
case 'quality_score':
return metrics.qualityScore || 0;
case 'success_rate':
return metrics.successRate;
default:
return 0;
}
}
evaluateCriteria(value, target, operator) {
switch (operator) {
case 'greater_than':
return value > target;
case 'less_than':
return value < target;
case 'equals':
return Math.abs(value - target) < 0.01;
default:
return false;
}
}
calculateConfidence(test) {
// Simple confidence calculation based on sample size and variance
const totalTests = test.variants.reduce((sum, v) => sum + v.testCount, 0);
const minSampleSize = test.config.sampleSize || 30;
if (totalTests < minSampleSize) {
return 0;
}
// Basic confidence calculation (simplified)
const sampleSizeConfidence = Math.min(totalTests / (minSampleSize * 2), 1);
const varianceConfidence = test.variants.length > 1 ? 0.8 : 0.5; // More variants = more confidence in winner
return sampleSizeConfidence * varianceConfidence;
}
}
exports.PromptVersionManager = PromptVersionManager;
/**
* Utility functions for prompt versioning
*/
/**
* Create a quick A/B test between two prompt templates
*/
function createQuickABTest(name, templateA, templateB, variables = {}, options = {}) {
const manager = new PromptVersionManager();
const versionA = manager.createVersion(`${name}_A`, templateA, variables, {
description: `${options.description || name} - Variant A`,
tags: ['ab-test', 'variant-a']
});
const versionB = manager.createVersion(`${name}_B`, templateB, variables, {
description: `${options.description || name} - Variant B`,
tags: ['ab-test', 'variant-b']
});
const testConfig = {
name,
description: options.description,
variants: [versionA, versionB],
trafficSplit: options.trafficSplit || [50, 50],
successCriteria: options.successCriteria || [
{ metric: 'success_rate', target: 90, operator: 'greater_than' }
]
};
return { manager, testConfig };
}
//# sourceMappingURL=prompt-versioning.js.map