@varia-bly/variably-sdk
Version:
Official JavaScript/TypeScript SDK for Variably feature flags, experimentation, LLM experiments with React hooks, and real-time dynamic configurations
1,060 lines • 44.1 kB
JavaScript
/**
* Variably LLM Client - Main implementation for LLM features
*/
import { EventEmitter } from 'events';
import { GraphQLClient } from './graphql-client';
import { ConsoleLogger } from './logger';
import { CacheManager } from './cache';
import { MetricsCollector } from './metrics';
export class VariablyLLMClient extends EventEmitter {
constructor(config) {
super();
this.config = config;
this.logger = config.logger || new ConsoleLogger('info');
this.metricsCollector = new MetricsCollector();
const variablyConfig = {
apiKey: config.apiKey,
baseUrl: config.baseUrl || 'https://graphql.variably.tech',
timeout: config.timeout || 30000,
retryAttempts: config.retryAttempts || 3,
};
this.graphqlClient = new GraphQLClient(variablyConfig, this.logger, this.metricsCollector);
this.cache = new CacheManager({
enabled: config.cacheConfig?.enabled ?? true,
ttl: config.cacheConfig?.ttl || 3600000, // 1 hour default
maxSize: config.cacheConfig?.maxSize || 100,
}, this.logger);
this.conversationContexts = new Map();
this.activeStreams = new Map();
this.sdkMetrics = {
promptExecutions: 0,
responseEvaluations: 0,
averageQualityScore: 0,
totalTokens: 0,
totalCost: 0,
providerMetrics: {
custom: { calls: 0, errors: 0, averageLatency: 0, tokensUsed: 0 },
openai: { calls: 0, errors: 0, averageLatency: 0, tokensUsed: 0 },
anthropic: { calls: 0, errors: 0, averageLatency: 0, tokensUsed: 0 },
google: { calls: 0, errors: 0, averageLatency: 0, tokensUsed: 0 },
azure: { calls: 0, errors: 0, averageLatency: 0, tokensUsed: 0 },
},
cacheMetrics: {
hits: 0,
misses: 0,
evictions: 0,
hitRate: 0,
},
errorBreakdown: {},
};
this.initializeEventHandlers();
}
/**
* Execute a prompt experiment with automatic variant selection and evaluation tracking
* RECOMMENDED: Use this method for prompt experiments as it properly saves evaluation data
*
* This method uses the backend's evaluation endpoint which:
* 1. Selects an experiment variant
* 2. Executes the LLM prompt
* 3. Saves evaluation data to the database
*
* @param experimentKey - The unique key for the experiment (not ID)
* @param request - The execution request with user context and variables
*/
async executePromptExperiment(experimentKey, request) {
const startTime = Date.now();
const executionId = this.generateExecutionId();
try {
this.logger.info('🚀 [SDK] Executing prompt experiment with evaluation tracking', {
executionId,
experimentKey,
userId: request.userContext.userId
});
// Call the backend evaluation endpoint that saves to database
const backendResponse = await this.graphqlClient.evaluatePromptExperiment(experimentKey, request.userContext, request.variables, request.metadata);
this.logger.info('✅ [SDK] Prompt experiment executed successfully', {
executionId: backendResponse.executionId,
variantUsed: backendResponse.variantUsed,
model: backendResponse.model,
tokensUsed: backendResponse.tokenUsage.totalTokens
});
// Map backend response to SDK response format
const llmResponse = {
content: backendResponse.content,
usage: backendResponse.tokenUsage,
model: backendResponse.model,
provider: backendResponse.provider,
finishReason: 'stop',
generatedAt: new Date(),
latencyMs: backendResponse.latencyMs,
};
// Calculate metrics
const metrics = {
latencyMs: Date.now() - startTime,
providerLatencyMs: backendResponse.latencyMs,
sdkOverheadMs: (Date.now() - startTime) - backendResponse.latencyMs,
tokensPerSecond: backendResponse.tokenUsage.totalTokens / (backendResponse.latencyMs / 1000),
cacheHit: false,
retries: 0,
};
// Build variant object
const variant = {
variantId: backendResponse.variantUsed,
name: backendResponse.variantUsed,
promptTemplate: '',
llmConfig: {
provider: backendResponse.provider,
model: backendResponse.model,
temperature: 0.7,
maxTokens: 2000,
},
};
// Build evaluation result if quality score is available
let evaluationResult;
if (backendResponse.qualityScore !== undefined) {
evaluationResult = {
overallScore: backendResponse.qualityScore,
dimensionScores: {},
ruleResults: [],
metadata: {
evaluationId: backendResponse.executionId,
evaluatedAt: new Date(),
evaluationMethod: 'automated',
},
};
}
// Build response
const response = {
executionId: backendResponse.executionId,
experimentId: backendResponse.experimentId,
variant,
response: llmResponse,
evaluation: evaluationResult,
metrics,
tracking: {
executionId: backendResponse.executionId,
experimentId: backendResponse.experimentId,
variantId: backendResponse.variantUsed,
userId: request.userContext.userId,
sessionId: request.sessionId,
trackedAt: new Date(),
properties: request.metadata,
},
};
// Update SDK metrics
this.updateSDKMetrics(llmResponse, evaluationResult);
// Emit success event
this.emit('promptExecuted', response);
return response;
}
catch (error) {
this.logger.error('❌ [SDK] Failed to execute prompt experiment', { executionId, experimentKey, error });
this.handleExecutionError(error, executionId);
throw error;
}
}
/**
* Execute a prompt experiment with automatic variant selection (LEGACY)
*
* NOTE: This method does NOT save evaluation data to the database.
* Use executePromptExperiment() instead for proper evaluation tracking.
*/
async executeLLMPrompt(request) {
const startTime = Date.now();
const executionId = this.generateExecutionId();
try {
this.logger.debug('Executing LLM prompt', { executionId, experimentId: request.experimentId });
this.logger.warn('⚠️ executeLLMPrompt does NOT save evaluations. Use executePromptExperiment() instead.');
// Check cache if enabled
const cacheKey = this.generateCacheKey(request);
const cachedResponse = await this.checkCache(cacheKey);
if (cachedResponse) {
this.logger.debug('Cache hit for prompt execution', { executionId });
this.updateCacheMetrics('hit');
return this.createResponseFromCache(cachedResponse, request, executionId);
}
this.updateCacheMetrics('miss');
// Get prompt variant from experiment
const variant = await this.getPromptVariant(request.experimentId, request.userContext);
// Compile prompt with variables
const compiledPrompt = this.compilePrompt(variant.promptTemplate, request.variables || {});
variant.compiledPrompt = compiledPrompt;
// Build conversation context if needed
const conversationContext = this.buildConversationContext(request);
// Execute prompt with provider
const llmResponse = await this.executeWithProvider(variant, request.llmConfig || {}, conversationContext);
// Evaluate response if requested
let evaluationResult;
if (request.includeEvaluation) {
evaluationResult = await this.evaluateResponseInternal(llmResponse.content, compiledPrompt, request.evaluationCriteria);
}
// Calculate metrics
const metrics = {
latencyMs: Date.now() - startTime,
providerLatencyMs: llmResponse.latencyMs,
sdkOverheadMs: (Date.now() - startTime) - llmResponse.latencyMs,
tokensPerSecond: llmResponse.usage.totalTokens / (llmResponse.latencyMs / 1000),
cacheHit: false,
retries: 0,
};
// Track metrics if enabled
if (request.trackMetrics) {
await this.trackExecutionMetrics(executionId, request, llmResponse, evaluationResult);
}
// Build response
const response = {
executionId,
experimentId: request.experimentId,
variant,
response: llmResponse,
evaluation: evaluationResult,
metrics,
tracking: {
executionId,
experimentId: request.experimentId,
variantId: variant.variantId,
userId: request.userContext.userId,
sessionId: request.sessionId,
trackedAt: new Date(),
properties: request.metadata,
},
};
// Cache successful response
if (this.config.cacheConfig?.enabled) {
await this.cacheResponse(cacheKey, response);
}
// Update SDK metrics
this.updateSDKMetrics(llmResponse, evaluationResult);
// Emit success event
this.emit('promptExecuted', response);
return response;
}
catch (error) {
this.logger.error('Failed to execute prompt', { executionId, error });
this.handleExecutionError(error, executionId);
throw error;
}
}
/**
* Evaluate a response against specified criteria
*/
async evaluateResponse(request) {
const startTime = Date.now();
const evaluationId = this.generateExecutionId();
try {
this.logger.debug('Evaluating response', { evaluationId });
// Perform evaluation
const evaluationResult = await this.evaluateResponseInternal(request.response, request.prompt, request.criteria, request.expectedOutput);
// Generate comparison if expected output provided
let comparison;
if (request.expectedOutput) {
comparison = await this.compareOutputs(request.response, request.expectedOutput);
}
// Generate recommendations
const recommendations = this.generateImprovementRecommendations(evaluationResult, comparison);
// Build response
const response = {
evaluation: evaluationResult,
comparison,
recommendations,
metadata: {
evaluationId,
evaluatedAt: new Date(),
processingTimeMs: Date.now() - startTime,
},
};
// Update metrics
this.sdkMetrics.responseEvaluations++;
// Emit event
this.emit('responseEvaluated', response);
return response;
}
catch (error) {
this.logger.error('Failed to evaluate response', { evaluationId, error });
throw error;
}
}
/**
* Track a success metric for an LLM prompt experiment
* Use this to track business outcomes (conversions, user satisfaction, etc.)
*
* @param params - The tracking parameters
* @param params.experimentId - The experiment ID (from executePromptExperiment response)
* @param params.metricName - Human-readable metric name (e.g., "conversion", "user_satisfaction")
* @param params.metricKey - Unique metric key for aggregation
* @param params.userId - The user ID who triggered the outcome
* @param params.sessionId - Optional session ID (recommended - from executePromptExperiment response)
* @param params.variantId - The variant UUID (from executePromptExperiment response)
* @param params.variantKey - Alternative: variant name/key (if variantId not available)
* @param params.value - Optional numeric value for the metric (default: 1.0)
* @param params.metadata - Optional additional metadata
*/
async trackSuccessMetric(params) {
try {
this.logger.info('📊 [SDK] Tracking success metric', {
experimentId: params.experimentId,
metricKey: params.metricKey,
userId: params.userId,
hasSessionId: !!params.sessionId,
hasVariantId: !!params.variantId,
});
await this.graphqlClient.trackSuccessMetric(params.experimentId, params.metricName, params.metricKey, params.userId, params.sessionId, params.variantId, params.variantKey, params.value !== undefined ? params.value : 1.0, params.metadata);
this.logger.info('✅ [SDK] Success metric tracked successfully', {
metricKey: params.metricKey,
});
// Emit event
this.emit('successMetricTracked', {
experimentId: params.experimentId,
metricKey: params.metricKey,
userId: params.userId,
});
}
catch (error) {
this.logger.error('❌ [SDK] Failed to track success metric', {
experimentId: params.experimentId,
metricKey: params.metricKey,
error,
});
throw error;
}
}
/**
* Execute prompts with streaming support
*/
async executeLLMPromptStreaming(request, options) {
const executionId = this.generateExecutionId();
const abortController = new AbortController();
this.activeStreams.set(executionId, abortController);
try {
// Get variant and compile prompt
const variant = await this.getPromptVariant(request.experimentId, request.userContext);
const compiledPrompt = this.compilePrompt(variant.promptTemplate, request.variables || {});
variant.compiledPrompt = compiledPrompt;
// Stream from provider
await this.streamFromProvider(variant, request.llmConfig || {}, options, abortController.signal);
}
catch (error) {
if (options.onError) {
options.onError(this.createLLMError(error));
}
throw error;
}
finally {
this.activeStreams.delete(executionId);
}
}
/**
* Execute multiple prompts in batch
*/
async executeBatchPrompts(request) {
const batchId = this.generateExecutionId();
const startTime = Date.now();
try {
this.logger.debug('Executing batch prompts', { batchId, count: request.requests.length });
const responses = [];
const options = request.options || {};
if (options.parallel) {
// Execute in parallel with concurrency control
const maxConcurrency = options.maxConcurrency || 5;
const chunks = this.chunkArray(request.requests, maxConcurrency);
for (const chunk of chunks) {
const chunkResponses = await Promise.allSettled(chunk.map(req => this.executeLLMPrompt(req)));
for (const result of chunkResponses) {
if (result.status === 'fulfilled') {
responses.push(result.value);
}
else {
const error = this.createLLMError(result.reason);
responses.push(error);
if (options.stopOnError) {
break;
}
}
}
if (options.stopOnError && responses.some(r => 'code' in r)) {
break;
}
}
}
else {
// Execute sequentially
for (const req of request.requests) {
try {
const response = await this.executeLLMPrompt(req);
responses.push(response);
}
catch (error) {
const llmError = this.createLLMError(error);
responses.push(llmError);
if (options.stopOnError) {
break;
}
}
}
}
// Calculate statistics
const successful = responses.filter(r => !('code' in r)).length;
const failed = responses.length - successful;
const successfulResponses = responses.filter(r => !('code' in r));
const statistics = {
total: request.requests.length,
successful,
failed,
averageLatencyMs: successfulResponses.reduce((sum, r) => sum + r.metrics.latencyMs, 0) / (successful || 1),
totalTokens: successfulResponses.reduce((sum, r) => sum + r.response.usage.totalTokens, 0),
totalCost: successfulResponses.reduce((sum, r) => sum + (r.response.usage.estimatedCost || 0), 0),
};
return {
batchId,
responses,
statistics,
};
}
catch (error) {
this.logger.error('Failed to execute batch prompts', { batchId, error });
throw error;
}
}
/**
* Manage conversation context
*/
addToConversation(sessionId, message) {
const context = this.conversationContexts.get(sessionId) || {
sessionId,
messages: [],
totalTokens: 0,
};
context.messages.push(message);
// Trim context if needed
const maxMessages = context.maxMessages || 10;
if (context.messages.length > maxMessages) {
context.messages = context.messages.slice(-maxMessages);
}
this.conversationContexts.set(sessionId, context);
}
getConversationContext(sessionId) {
return this.conversationContexts.get(sessionId);
}
clearConversation(sessionId) {
this.conversationContexts.delete(sessionId);
}
/**
* Export execution history
*/
async exportHistory(format, filter) {
// This would typically query from a persistence layer
// For now, returning a placeholder
return {
data: format.type === 'json' ? {} : '',
format,
count: 0,
exportedAt: new Date(),
};
}
/**
* Get SDK metrics
*/
getMetrics() {
return { ...this.sdkMetrics };
}
/**
* Clear cache
*/
clearCache() {
this.cache.clear();
this.logger.debug('LLM response cache cleared');
}
// Private helper methods
initializeEventHandlers() {
this.on('error', (error) => {
this.logger.error('LLM Client error', { error });
});
}
generateExecutionId() {
return `exec_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
generateCacheKey(request) {
const strategy = this.config.cacheConfig?.keyStrategy || 'prompt';
if (this.config.cacheConfig?.keyGenerator) {
return this.config.cacheConfig.keyGenerator(request);
}
const baseKey = `${request.experimentId}_${request.userContext.userId}`;
switch (strategy) {
case 'prompt':
return `${baseKey}_${JSON.stringify(request.variables)}`;
case 'prompt+config':
return `${baseKey}_${JSON.stringify(request.variables)}_${JSON.stringify(request.llmConfig)}`;
default:
return baseKey;
}
}
async checkCache(key) {
const cached = this.cache.get(key);
if (cached && cached.expiresAt > new Date()) {
cached.hitCount++;
return cached;
}
return null;
}
async cacheResponse(key, response) {
const ttl = (this.config.cacheConfig?.ttl || 3600) * 1000;
const cachedResponse = {
response: response.response,
key,
cachedAt: new Date(),
hitCount: 0,
expiresAt: new Date(Date.now() + ttl),
};
this.cache.set(key, cachedResponse);
}
createResponseFromCache(cached, request, executionId) {
return {
executionId,
experimentId: request.experimentId,
variant: {
variantId: 'cached',
name: 'Cached Response',
promptTemplate: '',
compiledPrompt: '',
llmConfig: {},
},
response: cached.response,
metrics: {
latencyMs: 0,
cacheHit: true,
retries: 0,
},
tracking: {
executionId,
experimentId: request.experimentId,
variantId: 'cached',
userId: request.userContext.userId,
trackedAt: new Date(),
},
};
}
async getPromptVariant(experimentId, userContext) {
// Use SDK-specific endpoint that accepts API key authentication
const query = `
query SdkGetPromptVariant($input: SDKPromptVariantInput!) {
sdkGetPromptVariant(input: $input) {
experiment {
id
trafficAllocation
}
selectedVariant {
id
name
prompt
weight
isControl
parameters
}
}
}
`;
const result = await this.graphqlClient.executeQuery({
query,
variables: {
input: {
experimentId,
context: {
userId: userContext.userId,
email: userContext.attributes?.email,
country: userContext.geo?.country,
language: userContext.preferences?.language,
platform: userContext.device?.platform,
version: userContext.attributes?.version,
ipAddress: userContext.attributes?.ipAddress,
userAgent: userContext.device?.browser,
sessionId: userContext.attributes?.sessionId,
attributes: userContext.attributes || {},
},
},
},
});
const data = result.sdkGetPromptVariant;
if (!data || !data.selectedVariant) {
throw new Error(`Failed to get prompt variant for experiment: ${experimentId}`);
}
const variant = data.selectedVariant;
// Validate that llmConfig exists in parameters
if (!variant.parameters?.llmConfig) {
throw new Error(`Variant "${variant.name}" (ID: ${variant.id}) is missing required llmConfig in parameters. ` +
`Please configure the LLM settings for this variant in the experiment.`);
}
// Map backend response to PromptVariant type
return {
variantId: variant.id,
name: variant.name,
promptTemplate: variant.prompt,
systemMessage: variant.parameters?.systemMessage || '',
llmConfig: variant.parameters.llmConfig,
trafficWeight: variant.weight,
metadata: variant.parameters?.metadata || {},
};
}
/**
* Production-grade variant selection using consistent hashing.
*
* Algorithm:
* 1. Generate a deterministic hash bucket (0-9999) for the user+experiment
* 2. Check if user falls within overall traffic allocation
* 3. Distribute traffic across variants based on their weights
*
* This ensures:
* - Same user always gets same variant (consistency)
* - Traffic is distributed according to configured weights
* - No bias in variant assignment
*/
selectVariantByTrafficAllocation(userId, experimentId, variants, overallTrafficAllocation) {
// Generate consistent hash bucket (0-9999) for this user+experiment
const bucketHash = this.hashToBucket(userId, experimentId, 10000);
// Check overall traffic allocation (default 100% if not specified)
const trafficPct = overallTrafficAllocation ?? 100;
if (trafficPct < 0 || trafficPct > 100) {
throw new Error(`Invalid traffic allocation: ${trafficPct}. Must be between 0-100`);
}
const trafficThreshold = (trafficPct / 100) * 10000;
if (bucketHash >= trafficThreshold) {
throw new Error(`User ${userId} not in traffic allocation (${trafficPct}%) for experiment ${experimentId}`);
}
// Calculate traffic weights for variants
const hasWeights = variants.some(v => v.trafficWeight !== undefined && v.trafficWeight !== null);
let weights;
if (hasWeights) {
// Use configured weights
weights = variants.map(v => v.trafficWeight ?? 0);
const totalWeight = weights.reduce((sum, w) => sum + w, 0);
if (totalWeight <= 0) {
throw new Error(`Invalid variant weights: total must be > 0, got ${totalWeight}`);
}
// Normalize weights to sum to traffic allocation
weights = weights.map(w => (w / totalWeight) * trafficThreshold);
}
else {
// Equal distribution across all variants
const weightPerVariant = trafficThreshold / variants.length;
weights = new Array(variants.length).fill(weightPerVariant);
}
// Select variant based on bucket falling into weight ranges
let cumulativeWeight = 0;
for (let i = 0; i < variants.length; i++) {
cumulativeWeight += weights[i];
if (bucketHash < cumulativeWeight) {
this.logger.debug('Variant selected', {
userId,
experimentId,
variantId: variants[i].variantId,
bucketHash,
cumulativeWeight
});
return variants[i];
}
}
// Should never reach here if weights are correct, but fail loudly if we do
throw new Error(`Variant selection algorithm error: bucket ${bucketHash} exceeded cumulative weight ${cumulativeWeight}`);
}
/**
* Hash function for consistent bucketing.
* Uses MurmurHash3-inspired algorithm for good distribution.
*
* @returns Integer in range [0, maxBucket)
*/
hashToBucket(userId, experimentId, maxBucket) {
const input = `${userId}:${experimentId}`;
let hash = 0;
for (let i = 0; i < input.length; i++) {
const char = input.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash | 0; // Convert to 32-bit integer
}
// Ensure positive and within bucket range
return Math.abs(hash) % maxBucket;
}
compilePrompt(template, variables) {
let compiled = template;
for (const [key, value] of Object.entries(variables)) {
const regex = new RegExp(`{{\\s*${key}\\s*}}`, 'g');
compiled = compiled.replace(regex, String(value));
}
// Check for any remaining placeholders
const remaining = compiled.match(/{{[^}]+}}/g);
if (remaining) {
this.logger.warn('Unresolved placeholders in prompt', { placeholders: remaining });
}
return compiled;
}
buildConversationContext(request) {
if (!request.sessionId) {
return undefined;
}
const existingContext = this.conversationContexts.get(request.sessionId);
const messages = existingContext?.messages || [];
if (request.conversationHistory) {
messages.push(...request.conversationHistory);
}
return {
sessionId: request.sessionId,
messages,
maxMessages: 10,
totalTokens: 0,
};
}
async executeWithProvider(variant, overrideConfig, context) {
const config = { ...variant.llmConfig, ...overrideConfig };
const startTime = Date.now();
// Build messages for provider
const messages = [];
if (variant.systemMessage) {
messages.push({ role: 'system', content: variant.systemMessage });
}
if (context) {
messages.push(...context.messages);
}
const prompt = variant.compiledPrompt || variant.promptTemplate;
if (!prompt) {
throw new Error(`No prompt available for variant ${variant.variantId}`);
}
messages.push({ role: 'user', content: prompt });
// Call provider-specific implementation
const providerResponse = await this.callProvider(config, messages);
// Format response
const response = {
content: providerResponse.content,
usage: providerResponse.usage,
model: config.model,
provider: config.provider,
finishReason: providerResponse.finishReason || 'stop',
providerMetadata: providerResponse.metadata,
generatedAt: new Date(),
latencyMs: Date.now() - startTime,
};
// Update conversation context if needed
if (context) {
this.addToConversation(context.sessionId, {
role: 'assistant',
content: response.content,
timestamp: response.generatedAt,
});
}
return response;
}
async callProvider(config, messages) {
console.log('🔧 callProvider invoked with config:', JSON.stringify(config, null, 2));
console.log('📝 Messages to send:', JSON.stringify(messages, null, 2));
const provider = config.provider?.toUpperCase();
const model = config.model;
console.log(`🎯 Using provider: ${provider}, model: ${model}`);
if (!provider) {
throw new Error('Provider is required in llmConfig');
}
if (!model) {
throw new Error('Model is required in llmConfig');
}
// Call GraphQL mutation for LLM execution via the public method
try {
const formattedMessages = messages.map(m => ({
role: m.role,
content: m.content
}));
console.log('📤 Calling GraphQL LLM execution via executeLLMPrompt');
const result = await this.graphqlClient.executeLLMPrompt(formattedMessages, provider, model, config.temperature || 0.7, config.maxTokens || 2000);
console.log('✅ GraphQL LLM response received:', JSON.stringify(result, null, 2));
// Map GraphQL response to SDK format
return {
content: result.content,
usage: {
promptTokens: result.tokenUsage.promptTokens,
completionTokens: result.tokenUsage.completionTokens,
totalTokens: result.tokenUsage.totalTokens,
estimatedCost: result.tokenUsage.estimatedCost || 0,
},
finishReason: result.finishReason,
metadata: {
provider: result.provider,
model: result.model,
},
};
}
catch (error) {
console.error('❌ LLM provider call failed:', error);
throw new Error(`LLM provider call failed: ${error instanceof Error ? error.message : String(error)}`);
}
}
async streamFromProvider(variant, overrideConfig, options, signal) {
const baseUrl = this.config.baseUrl || 'https://api.variably.io';
const url = `${baseUrl}/api/v1/internal/sdk/prompt-experiments/evaluate-stream`;
const startTime = Date.now();
const prompt = variant.compiledPrompt || variant.promptTemplate;
const body = JSON.stringify({
experiment_key: variant.variantId,
input_variables: { prompt },
context: { user_id: 'streaming-client' },
});
const response = await fetch(url, {
method: 'POST',
headers: {
'X-API-Key': this.config.apiKey,
'Content-Type': 'application/json',
'Accept': 'text/event-stream',
},
body,
signal,
});
if (!response.ok) {
throw new Error(`Streaming request failed: HTTP ${response.status}`);
}
if (!response.body) {
throw new Error('No response body for SSE stream');
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
let currentEventType = '';
let chunkIndex = 0;
let fullContent = '';
let tokenUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
while (true) {
const { done, value } = await reader.read();
if (done)
break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
if (line.startsWith('event: ')) {
currentEventType = line.slice(7).trim();
}
else if (line.startsWith('data: ')) {
try {
const parsed = JSON.parse(line.slice(6));
if (currentEventType === 'token') {
const tokenContent = parsed.content || '';
fullContent += tokenContent;
if (options.onChunk) {
options.onChunk({
content: tokenContent,
index: chunkIndex++,
isFinal: false,
timestamp: new Date(),
});
}
}
else if (currentEventType === 'metadata') {
tokenUsage = {
promptTokens: parsed.token_usage?.prompt_tokens || 0,
completionTokens: parsed.token_usage?.completion_tokens || 0,
totalTokens: parsed.token_usage?.total_tokens || 0,
};
}
else if (currentEventType === 'done') {
// Send final chunk
if (options.onChunk) {
options.onChunk({
content: '',
index: chunkIndex,
isFinal: true,
timestamp: new Date(),
});
}
}
}
catch {
// Non-JSON data, skip
}
currentEventType = '';
}
}
}
if (options.onComplete && !signal.aborted) {
options.onComplete({
content: fullContent,
usage: tokenUsage,
model: variant.llmConfig.model,
provider: variant.llmConfig.provider,
finishReason: 'stop',
generatedAt: new Date(),
latencyMs: Date.now() - startTime,
});
}
}
async evaluateResponseInternal(response, prompt, criteria, expectedOutput) {
// Default quality dimensions
const dimensions = criteria?.dimensions || [
{ name: 'relevance', weight: 0.3, method: 'automated' },
{ name: 'accuracy', weight: 0.3, method: 'automated' },
{ name: 'completeness', weight: 0.2, method: 'automated' },
{ name: 'clarity', weight: 0.2, method: 'automated' },
];
const dimensionScores = {};
// Evaluate each dimension
for (const dimension of dimensions) {
const score = await this.evaluateDimension(response, prompt, dimension);
dimensionScores[dimension.name] = score;
}
// Calculate overall score
const overallScore = dimensions.reduce((sum, dim) => {
return sum + (dimensionScores[dim.name] * (dim.weight || 1 / dimensions.length));
}, 0);
// Evaluate custom rules if provided
const ruleResults = criteria?.customRules
? await this.evaluateCustomRules(response, criteria.customRules)
: [];
return {
overallScore,
dimensionScores,
ruleResults,
metadata: {
evaluationId: this.generateExecutionId(),
evaluatedAt: new Date(),
evaluationMethod: 'automated',
},
};
}
async evaluateDimension(response, prompt, dimension) {
// Simplified evaluation logic
// In production, this would use more sophisticated methods
if (!dimension)
return 0.5;
switch (dimension.name) {
case 'relevance':
// Check if response relates to prompt
return prompt && response.length > 0 ? 0.8 : 0.5;
case 'accuracy':
// Would check factual accuracy
return 0.75;
case 'completeness':
// Check response length and structure
return response.length > 50 ? 0.8 : 0.6;
case 'clarity':
// Check readability and structure
return 0.7;
default:
return 0.5;
}
}
async evaluateCustomRules(response, rules) {
// Evaluate custom rules
return [];
}
async compareOutputs(actual, expected) {
// Compare outputs using various methods
return {
similarityScore: 0.85,
differences: [],
method: 'semantic',
};
}
generateImprovementRecommendations(evaluation, comparison) {
const recommendations = [];
// Generate recommendations based on low scores
for (const [dimension, score] of Object.entries(evaluation.dimensionScores)) {
if (score < 0.7) {
recommendations.push({
area: dimension,
recommendation: `Improve ${dimension} by focusing on...`,
priority: score < 0.5 ? 'high' : 'medium',
});
}
}
return recommendations;
}
async trackExecutionMetrics(executionId, request, response, evaluation) {
const event = {
name: 'llm_prompt_executed',
userId: request.userContext.userId,
properties: {
executionId,
experimentId: request.experimentId,
model: response.model,
provider: response.provider,
tokensUsed: response.usage.totalTokens,
cost: response.usage.estimatedCost,
latency: response.latencyMs,
qualityScore: evaluation?.overallScore,
...request.metadata,
},
timestamp: new Date(),
};
// Send to analytics
await this.graphqlClient.trackEvent(event);
}
updateSDKMetrics(response, evaluation) {
this.sdkMetrics.promptExecutions++;
this.sdkMetrics.totalTokens += response.usage.totalTokens;
this.sdkMetrics.totalCost += response.usage.estimatedCost || 0;
if (evaluation) {
const prevAvg = this.sdkMetrics.averageQualityScore;
const count = this.sdkMetrics.responseEvaluations;
this.sdkMetrics.averageQualityScore =
(prevAvg * count + evaluation.overallScore) / (count + 1);
}
// Update provider metrics
if (!this.sdkMetrics.providerMetrics[response.provider]) {
this.sdkMetrics.providerMetrics[response.provider] = {
calls: 0,
errors: 0,
averageLatency: 0,
tokensUsed: 0,
};
}
const providerMetrics = this.sdkMetrics.providerMetrics[response.provider];
providerMetrics.calls++;
providerMetrics.tokensUsed += response.usage.totalTokens;
providerMetrics.averageLatency =
(providerMetrics.averageLatency * (providerMetrics.calls - 1) + response.latencyMs) /
providerMetrics.calls;
}
updateCacheMetrics(type) {
if (type === 'hit') {
this.sdkMetrics.cacheMetrics.hits++;
}
else if (type === 'miss') {
this.sdkMetrics.cacheMetrics.misses++;
}
else {
this.sdkMetrics.cacheMetrics.evictions++;
}
const total = this.sdkMetrics.cacheMetrics.hits + this.sdkMetrics.cacheMetrics.misses;
this.sdkMetrics.cacheMetrics.hitRate =
total > 0 ? this.sdkMetrics.cacheMetrics.hits / total : 0;
}
createLLMError(error) {
return {
code: 'EXECUTION_ERROR',
message: error.message,
type: 'provider',
timestamp: new Date(),
};
}
handleExecutionError(error, executionId) {
const errorType = this.classifyError(error);
this.sdkMetrics.errorBreakdown[errorType] =
(this.sdkMetrics.errorBreakdown[errorType] || 0) + 1;
this.emit('error', {
executionId,
error,
type: errorType,
});
}
classifyError(error) {
if (error.message.includes('timeout'))
return 'timeout';
if (error.message.includes('rate'))
return 'rate_limit';
if (error.message.includes('token'))
return 'token_limit';
if (error.message.includes('network'))
return 'network';
return 'unknown';
}
chunkArray(array, size) {
const chunks = [];
for (let i = 0; i < array.length; i += size) {
chunks.push(array.slice(i, i + size));
}
return chunks;
}
}
//# sourceMappingURL=llm-client.js.map