@varia-bly/variably-sdk
Version:
Official JavaScript/TypeScript SDK for Variably feature flags, experimentation, LLM experiments with React hooks, and real-time dynamic configurations
654 lines • 20.1 kB
TypeScript
/**
* REST client for Variably SDK endpoints
*
* This client provides direct REST API access for:
* - Self-hosted agent communication
* - Prompt experimentation
* - Variant assignment
* - Metrics tracking
*/
import { UserContext } from './types';
export interface RESTClientConfig {
/** API key for authentication */
apiKey: string;
/** Base URL for the API (default: https://api.variably.io) */
baseUrl?: string;
/** Request timeout in milliseconds (default: 30000) */
timeout?: number;
/** Number of retry attempts (default: 3) */
retryAttempts?: number;
/** Enable debug logging (default: false) */
debug?: boolean;
}
export interface PromptEvaluationOptions {
/** Preferred LLM provider */
providerPreference?: string;
/** Minimum quality threshold (0-1) */
qualityThreshold?: number;
/** Maximum cost per evaluation in USD */
maxCostUsd?: number;
/** Custom metadata to include */
metadata?: Record<string, unknown>;
/** Evaluation context for grounding/coherence scoring */
evaluationContext?: EvaluationContext;
}
export interface PromptVariant {
/** Variant identifier */
variantId: string;
/** Variant name */
name: string;
/** The prompt template */
promptTemplate: string;
/** Whether this is the control variant */
isControl: boolean;
/** Traffic allocation percentage (0-100) */
trafficAllocation: number;
}
export interface PromptPerformance {
/** Overall quality score (0-1) */
overallScore: number;
/** Individual metric scores */
metricsScores: Record<string, number>;
/** Response latency in milliseconds */
latencyMs?: number;
/** Total tokens used */
tokenCount?: number;
/** Prompt tokens */
promptTokens?: number;
/** Completion tokens */
completionTokens?: number;
}
export interface PromptCost {
/** Cost in USD */
costUsd?: number;
/** LLM provider name */
providerName: string;
/** Model name */
modelName: string;
}
export interface PromptEvaluationResponse {
/** Experiment identifier */
experimentId: string;
/** Experiment key */
experimentKey: string;
/** Variant used for this evaluation */
variantId: string;
/** Variant name */
variantName: string;
/** Whether control variant was used */
isControl: boolean;
/** The generated response */
response: string;
/** The actual prompt sent to LLM */
prompt: string;
/** Response metadata */
metadata: Record<string, unknown>;
/** Performance metrics */
performance?: PromptPerformance;
/** Cost information */
cost?: PromptCost;
/** Timestamp of evaluation */
timestamp: string;
}
export interface VariantAssignment {
/** Experiment identifier */
experimentId: string;
/** Experiment key */
experimentKey: string;
/** Assigned variant */
variant: PromptVariant;
/** Assignment reason */
reason: 'random' | 'sticky' | 'override' | 'fallback';
/** Assignment timestamp */
timestamp: string;
}
export interface PromptExperiment {
/** Experiment identifier */
experimentId: string;
/** Experiment key (unique within project) */
experimentKey: string;
/** Human-readable name */
name: string;
/** Description */
description?: string;
/** Experiment status */
status: 'draft' | 'running' | 'paused' | 'completed';
/** Base prompt template */
basePrompt: string;
/** Target metrics for evaluation */
targetMetrics: string[];
/** Experiment configuration */
configuration: Record<string, unknown>;
/** Variants in this experiment */
variants: PromptVariant[];
/** Creation timestamp */
createdAt: string;
/** Last update timestamp */
updatedAt: string;
}
export interface PromptExperimentSummary {
/** Experiment identifier */
experimentId: string;
/** Experiment key */
experimentKey: string;
/** Human-readable name */
name: string;
/** Experiment status */
status: 'draft' | 'running' | 'paused' | 'completed';
/** Number of variants */
variantCount: number;
/** Total evaluations */
totalEvaluations: number;
/** Average quality score */
averageScore: number;
/** Total cost in USD */
totalCostUsd: number;
/** Creation timestamp */
createdAt: string;
}
export interface SubmitScoringRequest {
/** Experiment identifier */
experimentId: string;
/** Variant identifier */
variantId: string;
/** Project identifier */
projectId: string;
/** The prompt sent to the LLM */
prompt: string;
/** The LLM response to score */
response: string;
/** LLM provider (optional) */
provider?: string;
/** LLM model (optional) */
model?: string;
/** Temperature setting (optional) */
temperature?: number;
/** Response latency in ms (optional) */
latencyMs?: number;
/** Prompt token count (optional) */
promptTokens?: number;
/** Output token count (optional) */
outputTokens?: number;
/** Total token count (optional) */
totalTokens?: number;
/** Source of the request: 'cloud' or 'agent' */
source?: 'cloud' | 'agent';
/** Agent ID if from agent */
agentId?: string;
/** Evaluation context for grounding/coherence scoring */
evaluationContext?: EvaluationContext;
}
export interface ReferenceMaterial {
id: string;
content: string;
source?: string;
type?: string;
relevanceScore?: number;
}
export interface WorkflowStep {
role?: string;
step?: string;
input?: string;
output?: string;
content?: string;
}
export interface EvaluationContext {
referenceMaterials?: ReferenceMaterial[];
workflowHistory?: WorkflowStep[];
retrievalQuery?: string;
}
export interface ScoringSubmitResponse {
/** Unique request ID for polling */
requestId: string;
/** Current status */
status: 'pending' | 'processing' | 'completed' | 'failed';
/** Status message */
message: string;
/** Endpoint to poll for results */
pollEndpoint: string;
}
export interface ScoringResult {
/** Request identifier */
requestId: string;
/** Experiment identifier */
experimentId: string;
/** Variant identifier */
variantId: string;
/** Project identifier */
projectId: string;
/** Overall score (0-100) */
overallScore: number;
/** Quality score (35% weight) */
qualityScore: number;
/** Safety score (35% weight) */
safetyScore: number;
/** Semantic score (20% weight) */
semanticScore: number;
/** Advanced score (10% weight) */
advancedScore: number;
/** Detailed dimension scores (41 dimensions) */
dimensionScores: Record<string, number>;
/** Extracted metadata */
metadata?: ScoringMetadata;
/** Processing timestamp */
processedAt: string;
/** Processing time in ms */
processingMs: number;
/** Error message if failed */
error?: string;
}
export interface ScoringMetadata {
/** Prompt analysis */
prompt: {
charCount: number;
wordCount: number;
sentenceCount: number;
questionCount: number;
avgWordLength: number;
avgSentenceLength: number;
readabilityScore: number;
hasQuestion: boolean;
hasInstruction: boolean;
hasCodeRequest: boolean;
};
/** Response analysis */
response: {
charCount: number;
wordCount: number;
sentenceCount: number;
hasCode: boolean;
codeBlockCount: number;
avgWordLength: number;
avgSentenceLength: number;
readabilityScore: number;
vocabularyRichness: number;
containsPII: boolean;
toxicityScore: number;
sentimentScore: number;
sentimentLabel: string;
};
/** Comparative analysis */
comparative: {
responsePromptRatio: number;
keywordOverlap: number;
topicAlignment: number;
formatCompliance: number;
};
}
export interface ScoringRequest {
/** Internal ID */
id: string;
/** Request ID for tracking */
requestId: string;
/** Experiment identifier */
experimentId: string;
/** Variant identifier */
variantId: string;
/** Project identifier */
projectId: string;
/** Current status */
status: 'pending' | 'processing' | 'completed' | 'failed';
/** Source: 'cloud' or 'agent' */
source: string;
/** Agent ID if applicable */
agentId?: string;
/** Scoring result (if completed) */
result?: ScoringResult;
/** Error message (if failed) */
error?: string;
/** Creation timestamp */
createdAt: string;
/** Last update timestamp */
updatedAt: string;
/** Processing timestamp (if processed) */
processedAt?: string;
}
export interface ScoringStats {
/** Total requests */
totalRequests: number;
/** Pending requests */
pendingRequests: number;
/** Completed requests */
completedRequests: number;
/** Failed requests */
failedRequests: number;
/** Average processing time in ms */
avgProcessingMs: number;
/** Average overall score */
avgOverallScore: number;
}
export interface TrackMetricRequest {
/** Experiment identifier */
experimentId: string;
/** User identifier */
userId: string;
/** Metric key (e.g., 'video_completion_rate', 'watch_time') */
metricKey: string;
/** Metric value */
value: number;
/** Variant identifier (optional, for attribution) */
variantId?: string;
/** Session identifier */
sessionId?: string;
/** Custom metadata */
metadata?: Record<string, unknown>;
/** Timestamp (default: now) */
timestamp?: string;
}
export interface TrackEventRequest {
/** Event name/key */
eventKey: string;
/** User identifier */
userId: string;
/** Event type */
eventType?: 'metric_event' | 'conversion' | 'engagement';
/** Event value (optional) */
value?: number;
/** Associated experiment ID */
experimentId?: string;
/** Associated variant ID */
variantId?: string;
/** Session identifier */
sessionId?: string;
/** Custom metadata */
metadata?: Record<string, unknown>;
/** Timestamp (default: now) */
timestamp?: string;
}
export interface SuccessMetric {
/** Metric key */
metricKey: string;
/** Display name */
displayName: string;
/** Calculation method */
calculationMethod: 'sum' | 'average' | 'count' | 'ratio';
/** Whether this is the primary metric */
isPrimary: boolean;
}
export declare class VariablyRESTClient {
private config;
private logger;
constructor(config: RESTClientConfig);
/**
* Evaluate a prompt through the experimentation system
*/
evaluatePrompt(experimentKey: string, inputVariables: Record<string, unknown>, userContext: UserContext, options?: PromptEvaluationOptions): Promise<PromptEvaluationResponse>;
/**
* Get variant assignment for a prompt experiment
*/
getPromptVariant(experimentKey: string, userContext: UserContext): Promise<VariantAssignment>;
/**
* Get list of available prompt experiments
*/
getPromptExperiments(filters?: {
status?: string;
page?: number;
pageSize?: number;
}): Promise<{
experiments: PromptExperimentSummary[];
total: number;
}>;
/**
* Get details for a specific prompt experiment by key
*/
getPromptExperiment(experimentKey: string): Promise<PromptExperiment>;
/**
* Get details for a specific prompt experiment by ID
*/
getPromptExperimentById(experimentId: string): Promise<PromptExperiment>;
/**
* Track an experiment metric
*/
trackMetric(request: TrackMetricRequest): Promise<{
success: boolean;
}>;
/**
* Track a single event
*/
trackEvent(request: TrackEventRequest): Promise<{
success: boolean;
}>;
/**
* Track multiple events in batch
*/
trackEventBatch(events: TrackEventRequest[]): Promise<{
success: boolean;
count: number;
}>;
/**
* Get success metrics for an experiment
*/
getExperimentSuccessMetrics(experimentId: string): Promise<SuccessMetric[]>;
/**
* Submit a scoring request for async processing
*
* This submits a prompt-response pair for quality evaluation using
* Variably's 41-dimension scoring system. The scoring is performed
* asynchronously - use getScoringRequest() to poll for results.
*
* @param request - The scoring request with prompt/response
* @returns Submit response with request ID for polling
*/
submitScoringRequest(request: SubmitScoringRequest): Promise<ScoringSubmitResponse>;
/**
* Get a scoring request by ID (poll for results)
*
* @param requestId - The scoring request ID
* @returns The scoring request with status and results (if completed)
*/
getScoringRequest(requestId: string): Promise<ScoringRequest>;
/**
* Get scoring requests for an experiment
*
* @param experimentId - The experiment ID
* @param options - Pagination options
* @returns List of scoring requests
*/
getScoringRequestsByExperiment(experimentId: string, options?: {
limit?: number;
offset?: number;
}): Promise<{
requests: ScoringRequest[];
limit: number;
offset: number;
}>;
/**
* Get scoring statistics for a project
*
* @param projectId - The project ID
* @returns Scoring statistics
*/
getScoringStats(projectId: string): Promise<ScoringStats>;
/**
* Submit scoring request and wait for results
*
* This is a convenience method that submits a scoring request and
* polls until the result is ready or timeout is reached.
*
* @param request - The scoring request
* @param options - Polling options
* @returns The completed scoring result
*/
submitAndWaitForScoring(request: SubmitScoringRequest, options?: {
/** Maximum time to wait in ms (default: 60000) */
timeoutMs?: number;
/** Polling interval in ms (default: 1000) */
pollIntervalMs?: number;
}): Promise<ScoringRequest>;
/**
* Convert raw scoring request response to typed object
*/
private convertScoringRequest;
/**
* Convert raw scoring result to typed object
*/
private convertScoringResult;
/**
* Evaluate a prompt experiment with streaming via Server-Sent Events.
*
* Connects to the SSE endpoint and invokes callbacks as tokens arrive.
* Use this for real-time token-by-token display in chat UIs.
*
* @param experimentKey - Experiment key
* @param inputVariables - Template variables
* @param userContext - User context for variant selection
* @param callbacks - Streaming callbacks
* @param options - Evaluation options
* @returns AbortController to cancel the stream
*/
evaluatePromptStream(experimentKey: string, inputVariables: Record<string, unknown>, userContext: UserContext, callbacks: {
/** Called for each token chunk */
onToken: (token: string) => void;
/** Called when variant info is received (before tokens) */
onVariant?: (variant: {
experimentId: string;
variantId: string;
variantKey: string;
model: string;
provider: string;
}) => void;
/** Called when metadata is received (after all tokens) */
onMetadata?: (metadata: {
executionId: string;
tokenUsage: {
promptTokens: number;
completionTokens: number;
totalTokens: number;
};
latencyMs: number;
qualityScore?: number;
}) => void;
/** Called when the stream completes */
onComplete?: () => void;
/** Called on error */
onError?: (error: Error) => void;
}, options?: PromptEvaluationOptions): AbortController;
/**
* Assign a variant to a user for an experiment
*/
assignVariant(experimentId: string, userContext: UserContext): Promise<VariantAssignment>;
/**
* Evaluate a feature flag via REST API
*/
evaluateFlag(flagKey: string, userContext: UserContext): Promise<{
flagKey: string;
value: unknown;
reason: string;
ruleId?: string;
experimentId?: string;
variantId?: string;
}>;
/**
* Evaluate a feature gate via REST API
*/
evaluateGate(gateKey: string, userContext: UserContext): Promise<{
gateKey: string;
value: boolean;
reason: string;
ruleId?: string;
experimentId?: string;
variantId?: string;
successMetrics?: string[];
}>;
/**
* Execute an LLM prompt directly
*/
executeLLMPrompt(experimentKey: string, inputVariables: Record<string, unknown>, userContext: UserContext, options?: {
provider?: string;
model?: string;
temperature?: number;
maxTokens?: number;
}): Promise<{
response: string;
variantId: string;
experimentId: string;
usage?: {
promptTokens: number;
completionTokens: number;
totalTokens: number;
};
latencyMs: number;
}>;
/**
* Evaluate a dynamic config
*/
evaluateConfig(configKey: string, userContext: UserContext): Promise<{
configKey: string;
value: unknown;
reason: string;
etag?: string;
version?: number;
}>;
/**
* Evaluate multiple dynamic configs in batch
*/
evaluateConfigBatch(configKeys: string[], userContext: UserContext): Promise<Record<string, {
configKey: string;
value: unknown;
reason: string;
etag?: string;
version?: number;
}>>;
private request;
private convertEvaluationContext;
private convertUserContext;
private convertPromptEvaluationResponse;
/**
* Log a prompt/response pair for 43-dimension evaluation (Observe Mode).
*
* No experiment creation needed. Just log your prompts and responses
* and see quality scores in the dashboard.
*
* @example
* ```ts
* const result = await client.log({
* prompt: "What is TypeScript?",
* response: "TypeScript is a typed superset of JavaScript.",
* provider: "openai",
* model: "gpt-4",
* });
* console.log(result.observationId);
* ```
*/
log(params: import('./llm-types').LogParams): Promise<import('./llm-types').LogResult>;
private normalizeConfig;
}
/**
* Create a new REST client
*/
export declare function createRESTClient(config: RESTClientConfig): VariablyRESTClient;
/**
* Create a REST client from environment variables
*/
export declare function createRESTClientFromEnv(): VariablyRESTClient;
/**
* Builder class for constructing prompt input variables
*/
export declare class PromptContextBuilder {
private variables;
/**
* Set product information
*/
product(name: string, price?: string, features?: string[]): this;
/**
* Set customer information
*/
customer(name?: string, segment?: string, preferences?: string[]): this;
/**
* Set tone for the prompt
*/
tone(tone: 'professional' | 'casual' | 'friendly' | 'urgent' | 'empathetic'): this;
/**
* Set target audience
*/
audience(audience: string): this;
/**
* Set a custom variable
*/
custom(key: string, value: unknown): this;
/**
* Build the variables object
*/
build(): Record<string, unknown>;
}
//# sourceMappingURL=rest-client.d.ts.map