@varia-bly/variably-sdk
Version:
Official JavaScript/TypeScript SDK for Variably feature flags, experimentation, LLM experiments with React hooks, and real-time dynamic configurations
649 lines • 19.5 kB
TypeScript
/**
* Type definitions for Variably LLM SDK features
*/
export type LLMProvider = 'openai' | 'anthropic' | 'google' | 'azure' | 'custom';
export interface LLMConfig {
/** LLM provider to use */
provider: LLMProvider;
/** Model identifier (e.g., 'gpt-4', 'claude-3-opus') */
model: string;
/** Provider API key (optional if using server-side configuration) */
apiKey?: string;
/** Provider endpoint URL (for custom or Azure providers) */
endpoint?: string;
/** Maximum tokens in response */
maxTokens?: number;
/** Temperature for response generation (0-2) */
temperature?: number;
/** Top-p sampling parameter */
topP?: number;
/** Frequency penalty (-2.0 to 2.0) */
frequencyPenalty?: number;
/** Presence penalty (-2.0 to 2.0) */
presencePenalty?: number;
/** Stop sequences */
stopSequences?: string[];
/** Request timeout in milliseconds */
timeout?: number;
/** Enable streaming responses */
streaming?: boolean;
/** Custom headers for provider requests */
customHeaders?: Record<string, string>;
}
export interface PromptExecutionRequest {
/** Experiment ID for the prompt experiment */
experimentId: string;
/** User context for variant selection */
userContext: LLMUserContext;
/** Input variables for the prompt template */
variables?: Record<string, any>;
/** Override LLM configuration */
llmConfig?: Partial<LLMConfig>;
/** Include evaluation in response */
includeEvaluation?: boolean;
/** Custom evaluation criteria */
evaluationCriteria?: EvaluationCriteria;
/** Enable automatic metric tracking */
trackMetrics?: boolean;
/** Session ID for conversation continuity */
sessionId?: string;
/** Previous conversation history */
conversationHistory?: ConversationMessage[];
/** Metadata for tracking */
metadata?: Record<string, any>;
}
export interface PromptExecutionResponse {
/** Unique execution ID */
executionId: string;
/** Experiment ID */
experimentId: string;
/** Selected variant */
variant: PromptVariant;
/** Generated response */
response: LLMResponse;
/** Evaluation results (if requested) */
evaluation?: EvaluationResult;
/** Performance metrics */
metrics: PerformanceMetrics;
/** Tracking information */
tracking: TrackingInfo;
/** Error details (if any) */
error?: LLMError;
}
export interface LLMUserContext {
/** Unique user identifier */
userId: string;
/** User attributes for targeting */
attributes?: Record<string, any>;
/** User segments */
segments?: string[];
/** Geographic information */
geo?: {
country?: string;
region?: string;
city?: string;
};
/** Device/platform information */
device?: {
type?: 'desktop' | 'mobile' | 'tablet';
platform?: string;
browser?: string;
};
/** User preferences */
preferences?: {
language?: string;
responseStyle?: 'concise' | 'detailed' | 'technical' | 'casual';
outputFormat?: 'text' | 'markdown' | 'json' | 'html';
};
}
export interface PromptVariant {
/** Variant ID */
variantId: string;
/** Variant name */
name: string;
/** Prompt template */
promptTemplate: string;
/** Compiled prompt with variables */
compiledPrompt?: string;
/** System message (if applicable) */
systemMessage?: string;
/** LLM configuration for this variant */
llmConfig: LLMConfig;
/** Traffic weight for this variant (used in allocation) */
trafficWeight?: number;
/** Variant metadata */
metadata?: Record<string, any>;
}
export interface LLMResponse {
/** Generated text content */
content: string;
/** Token usage information */
usage: TokenUsage;
/** Model used for generation */
model: string;
/** Provider used */
provider: LLMProvider;
/** Finish reason */
finishReason: 'stop' | 'length' | 'content_filter' | 'error';
/** Response metadata from provider */
providerMetadata?: Record<string, any>;
/** Generated at timestamp */
generatedAt: Date;
/** Response latency in milliseconds */
latencyMs: number;
}
export interface TokenUsage {
/** Tokens in the prompt */
promptTokens: number;
/** Tokens in the completion */
completionTokens: number;
/** Total tokens used */
totalTokens: number;
/** Estimated cost in USD */
estimatedCost?: number;
}
export interface EvaluationCriteria {
/** Quality dimensions to evaluate */
dimensions: QualityDimension[];
/** Custom evaluation rules */
customRules?: EvaluationRule[];
/** Minimum acceptable scores */
thresholds?: Record<string, number>;
/** Use human review */
requireHumanReview?: boolean;
/** Evaluation provider (for automated evaluation) */
evaluationProvider?: 'openai' | 'anthropic' | 'custom';
/** Evaluation model to use */
evaluationModel?: string;
}
export interface QualityDimension {
/** Dimension name */
name: string;
/** Description of what to evaluate */
description?: string;
/** Weight in overall score (0-1) */
weight?: number;
/** Evaluation method */
method: 'automated' | 'human' | 'hybrid';
/** Custom evaluation prompt (for automated) */
evaluationPrompt?: string;
}
export interface EvaluationRule {
/** Rule ID */
ruleId: string;
/** Rule type */
type: 'regex' | 'contains' | 'length' | 'custom';
/** Rule configuration */
config: Record<string, any>;
/** Score impact */
scoreImpact: number;
/** Error message if rule fails */
errorMessage?: string;
}
export interface EvaluationResult {
/** Overall quality score (0-1) */
overallScore: number;
/** Individual dimension scores */
dimensionScores: Record<string, number>;
/** Rule evaluation results */
ruleResults?: RuleResult[];
/** Evaluation metadata */
metadata: {
evaluationId: string;
evaluatedAt: Date;
evaluationMethod: string;
evaluationModel?: string;
};
/** Human review status */
humanReview?: {
required: boolean;
completed: boolean;
reviewerId?: string;
reviewedAt?: Date;
feedback?: string;
overrideScore?: number;
};
/** Suggestions for improvement */
suggestions?: string[];
}
export interface RuleResult {
/** Rule ID */
ruleId: string;
/** Whether the rule passed */
passed: boolean;
/** Score impact applied */
scoreImpact: number;
/** Error message (if failed) */
errorMessage?: string;
/** Rule execution details */
details?: Record<string, any>;
}
export interface ResponseEvaluationRequest {
/** Response to evaluate */
response: string;
/** Original prompt */
prompt?: string;
/** Expected output (for comparison) */
expectedOutput?: string;
/** Evaluation criteria */
criteria: EvaluationCriteria;
/** Context for evaluation */
context?: Record<string, any>;
/** User context */
userContext?: LLMUserContext;
/** Execution ID (for linking to prompt execution) */
executionId?: string;
}
export interface ResponseEvaluationResponse {
/** Evaluation result */
evaluation: EvaluationResult;
/** Comparison with expected output */
comparison?: ComparisonResult;
/** Improvement recommendations */
recommendations?: ImprovementRecommendation[];
/** Evaluation metadata */
metadata: {
evaluationId: string;
evaluatedAt: Date;
processingTimeMs: number;
};
}
export interface ComparisonResult {
/** Similarity score (0-1) */
similarityScore: number;
/** Differences found */
differences: Difference[];
/** Comparison method used */
method: 'semantic' | 'exact' | 'fuzzy';
}
export interface Difference {
/** Type of difference */
type: 'missing' | 'extra' | 'incorrect' | 'formatting';
/** Description of the difference */
description: string;
/** Expected value */
expected?: string;
/** Actual value */
actual?: string;
/** Severity of the difference */
severity: 'low' | 'medium' | 'high';
}
export interface ImprovementRecommendation {
/** Area of improvement */
area: string;
/** Specific recommendation */
recommendation: string;
/** Priority level */
priority: 'low' | 'medium' | 'high';
/** Example of improved output */
example?: string;
}
export interface StreamingOptions {
/** Enable streaming */
enabled: boolean;
/** Chunk processing callback */
onChunk?: (chunk: StreamChunk) => void;
/** Stream complete callback */
onComplete?: (response: LLMResponse) => void;
/** Error callback */
onError?: (error: LLMError) => void;
/** Buffer size for chunks */
bufferSize?: number;
}
export interface StreamChunk {
/** Chunk content */
content: string;
/** Chunk index */
index: number;
/** Is final chunk */
isFinal: boolean;
/** Timestamp */
timestamp: Date;
}
export interface ConversationMessage {
/** Message role */
role: 'system' | 'user' | 'assistant';
/** Message content */
content: string;
/** Message metadata */
metadata?: Record<string, any>;
/** Message timestamp */
timestamp?: Date;
}
export interface ConversationContext {
/** Session ID */
sessionId: string;
/** Conversation history */
messages: ConversationMessage[];
/** Context window size */
maxMessages?: number;
/** Total tokens in context */
totalTokens?: number;
/** Conversation metadata */
metadata?: Record<string, any>;
}
export interface LLMCacheConfig {
/** Enable response caching */
enabled: boolean;
/** Cache TTL in seconds */
ttl?: number;
/** Maximum cache size */
maxSize?: number;
/** Cache key strategy */
keyStrategy?: 'prompt' | 'prompt+config' | 'custom';
/** Custom cache key generator */
keyGenerator?: (request: PromptExecutionRequest) => string;
/** Cache storage backend */
storage?: 'memory' | 'localStorage' | 'custom';
}
export interface CachedResponse {
/** Cached response */
response: LLMResponse;
/** Cache key */
key: string;
/** Cached at timestamp */
cachedAt: Date;
/** Cache hit count */
hitCount: number;
/** Cache expiry */
expiresAt: Date;
}
export interface LLMError {
/** Error code */
code: string;
/** Error message */
message: string;
/** Error type */
type: 'provider' | 'network' | 'validation' | 'evaluation' | 'rate_limit' | 'quota';
/** Provider-specific error details */
providerError?: any;
/** Retry information */
retry?: {
shouldRetry: boolean;
retryAfter?: number;
attemptNumber?: number;
maxAttempts?: number;
};
/** Error timestamp */
timestamp: Date;
}
export interface PerformanceMetrics {
/** Total latency in milliseconds */
latencyMs: number;
/** Time to first token (for streaming) */
timeToFirstTokenMs?: number;
/** Tokens per second */
tokensPerSecond?: number;
/** Provider latency */
providerLatencyMs?: number;
/** SDK overhead */
sdkOverheadMs?: number;
/** Cache hit */
cacheHit: boolean;
/** Network retries */
retries: number;
}
export interface TrackingInfo {
/** Execution ID */
executionId: string;
/** Experiment ID */
experimentId: string;
/** Variant ID */
variantId: string;
/** User ID */
userId: string;
/** Session ID */
sessionId?: string;
/** Tracked at timestamp */
trackedAt: Date;
/** Custom event properties */
properties?: Record<string, any>;
}
export interface LLMSDKMetrics {
/** Total prompt executions */
promptExecutions: number;
/** Total response evaluations */
responseEvaluations: number;
/** Average quality score */
averageQualityScore: number;
/** Total tokens consumed */
totalTokens: number;
/** Total cost incurred */
totalCost: number;
/** Provider-specific metrics */
providerMetrics: Record<LLMProvider, {
calls: number;
errors: number;
averageLatency: number;
tokensUsed: number;
}>;
/** Cache metrics */
cacheMetrics: {
hits: number;
misses: number;
evictions: number;
hitRate: number;
};
/** Error breakdown */
errorBreakdown: Record<string, number>;
}
export interface BatchPromptRequest {
/** Multiple prompt execution requests */
requests: PromptExecutionRequest[];
/** Batch processing options */
options?: {
/** Process in parallel */
parallel?: boolean;
/** Maximum concurrent requests */
maxConcurrency?: number;
/** Stop on first error */
stopOnError?: boolean;
/** Batch timeout */
timeoutMs?: number;
};
}
export interface BatchPromptResponse {
/** Batch ID */
batchId: string;
/** Individual responses */
responses: (PromptExecutionResponse | LLMError)[];
/** Batch statistics */
statistics: {
total: number;
successful: number;
failed: number;
averageLatencyMs: number;
totalTokens: number;
totalCost: number;
};
}
export interface ExportFormat {
/** Format type */
type: 'json' | 'csv' | 'markdown';
/** Include metadata */
includeMetadata?: boolean;
/** Include evaluation results */
includeEvaluation?: boolean;
/** Custom fields to export */
fields?: string[];
}
export interface ExportResult {
/** Exported data */
data: string | object;
/** Export format */
format: ExportFormat;
/** Number of items exported */
count: number;
/** Export timestamp */
exportedAt: Date;
}
export interface LogParams {
/** The prompt sent to the LLM */
prompt: string;
/** The LLM's response text */
response: string;
/** Prior conversation turns for cross-turn coherence scoring */
conversationHistory?: ConversationMessage[];
/** Retrieved RAG chunks for grounding/hallucination scoring */
referenceMaterials?: Array<{
id: string;
content: string;
source?: string;
type?: string;
relevanceScore?: number;
}>;
/** The query sent to the retriever */
retrievalQuery?: string;
/** Tags for grouping observations (e.g. ["production", "chatbot"]) */
tags?: string[];
/** End-user ID for per-user analytics */
userId?: string;
/** Session ID for multi-turn tracking */
sessionId?: string;
/** LLM provider name (e.g. "openai", "anthropic") */
provider?: string;
/** Model name (e.g. "gpt-4", "claude-3-opus") */
model?: string;
/** LLM call latency in milliseconds */
latencyMs?: number;
/** Number of input tokens */
promptTokens?: number;
/** Number of output tokens */
completionTokens?: number;
/** Arbitrary key-value metadata */
metadata?: Record<string, unknown>;
}
export interface LogResult {
/** Unique observation ID */
observationId: string;
/** Auto-created experiment ID */
experimentId: string;
/** Status — always "queued" (scoring is async) */
status: string;
}
export type BusinessOutcomeType = 'conversion' | 'revenue' | 'user_satisfaction' | 'task_completion' | 'engagement' | 'retention' | 'custom';
export interface BusinessOutcome {
/** Type of business outcome */
type: BusinessOutcomeType;
/** Outcome name (e.g., 'purchase_completed', 'signup_completed') */
name: string;
/** Numeric value associated with the outcome */
value?: number;
/** Currency code for revenue outcomes */
currency?: string;
/** User ID who triggered the outcome */
userId: string;
/** Execution ID that led to this outcome */
executionId?: string;
/** Experiment ID associated with this outcome */
experimentId?: string;
/** Variant ID that was shown */
variantId?: string;
/** Session ID */
sessionId?: string;
/** Additional outcome properties */
properties?: Record<string, any>;
/** Timestamp of the outcome */
timestamp?: Date;
}
export interface UserFeedback {
/** Execution ID being rated */
executionId: string;
/** User ID providing feedback */
userId: string;
/** Numeric rating (1-5, or custom scale) */
rating?: number;
/** Thumbs up/down */
sentiment?: 'positive' | 'negative' | 'neutral';
/** Free-form feedback text */
comment?: string;
/** Specific feedback category */
category?: 'accuracy' | 'helpfulness' | 'relevance' | 'quality' | 'other';
/** Additional feedback properties */
properties?: Record<string, any>;
/** Feedback timestamp */
timestamp?: Date;
}
export interface ConversionEvent {
/** Event name (e.g., 'add_to_cart', 'purchase', 'signup') */
eventName: string;
/** User ID */
userId: string;
/** Execution ID that influenced this conversion */
executionId?: string;
/** Experiment ID */
experimentId?: string;
/** Variant ID */
variantId?: string;
/** Revenue amount if applicable */
revenue?: number;
/** Currency code */
currency?: string;
/** Event properties */
properties?: Record<string, any>;
/** Funnel step (for funnel analysis) */
funnelStep?: string;
/** Event timestamp */
timestamp?: Date;
}
export interface BusinessMetricsRequest {
/** Experiment ID */
experimentId: string;
/** Time range start */
startDate?: Date;
/** Time range end */
endDate?: Date;
/** Group by variant */
groupByVariant?: boolean;
/** Include conversion rates */
includeConversions?: boolean;
/** Include revenue metrics */
includeRevenue?: boolean;
/** Include satisfaction scores */
includeSatisfaction?: boolean;
}
export interface BusinessMetricsResponse {
/** Experiment ID */
experimentId: string;
/** Time period covered */
period: {
start: Date;
end: Date;
};
/** Overall business metrics */
overall: {
/** Total conversions */
conversions: number;
/** Conversion rate */
conversionRate: number;
/** Total revenue */
totalRevenue?: number;
/** Average revenue per user */
averageRevenuePerUser?: number;
/** Average satisfaction score */
averageSatisfaction?: number;
/** Task completion rate */
taskCompletionRate?: number;
/** Engagement rate */
engagementRate?: number;
};
/** Metrics by variant */
byVariant?: Record<string, {
variantId: string;
variantName: string;
conversions: number;
conversionRate: number;
totalRevenue?: number;
averageRevenuePerUser?: number;
averageSatisfaction?: number;
taskCompletionRate?: number;
engagementRate?: number;
/** Statistical significance */
isSignificant?: boolean;
/** P-value for variant comparison */
pValue?: number;
}>;
/** Winner variant ID (if statistically significant) */
winnerVariantId?: string;
}
//# sourceMappingURL=llm-types.d.ts.map