UNPKG

@varia-bly/variably-sdk

Version:

Official JavaScript/TypeScript SDK for Variably feature flags, experimentation, LLM experiments with React hooks, and real-time dynamic configurations

926 lines 39.1 kB
/** * REST client for Variably SDK endpoints * * This client provides direct REST API access for: * - Self-hosted agent communication * - Prompt experimentation * - Variant assignment * - Metrics tracking */ import { ConsoleLogger } from './logger'; import { NetworkError, AuthenticationError, ValidationError } from './errors'; // ============================================================================= // REST Client Implementation // ============================================================================= export class VariablyRESTClient { constructor(config) { this.config = this.normalizeConfig(config); this.logger = new ConsoleLogger(this.config.debug ? 'debug' : 'info'); this.logger.info('VariablyRESTClient initialized', { baseUrl: this.config.baseUrl }); } // =========================================================================== // Prompt Experimentation // =========================================================================== /** * Evaluate a prompt through the experimentation system */ async evaluatePrompt(experimentKey, inputVariables, userContext, options = {}) { const response = await this.request('POST', '/internal/sdk/prompts/evaluate', { experiment_key: experimentKey, input_variables: inputVariables, context: this.convertUserContext(userContext), provider_preference: options.providerPreference, quality_threshold: options.qualityThreshold, max_cost_usd: options.maxCostUsd, metadata: options.metadata, evaluation_context: options.evaluationContext ? this.convertEvaluationContext(options.evaluationContext) : undefined }); return this.convertPromptEvaluationResponse(response); } /** * Get variant assignment for a prompt experiment */ async getPromptVariant(experimentKey, userContext) { const response = await this.request('POST', '/internal/sdk/prompt-experiments/get-variant', { experiment_key: experimentKey, context: this.convertUserContext(userContext) }); return { experimentId: response.experiment_id, experimentKey: response.experiment_key, variant: { variantId: response.variant.variant_id, name: response.variant.name, promptTemplate: response.variant.prompt_template, isControl: response.variant.is_control, trafficAllocation: response.variant.traffic_allocation }, reason: response.reason, timestamp: response.timestamp }; } /** * Get list of available prompt experiments */ async getPromptExperiments(filters) { const queryParams = new URLSearchParams(); if (filters?.status) queryParams.set('status', filters.status); if (filters?.page) queryParams.set('page', String(filters.page)); if (filters?.pageSize) queryParams.set('page_size', String(filters.pageSize)); const endpoint = `/internal/sdk/prompts/experiments${queryParams.toString() ? `?${queryParams}` : ''}`; const response = await this.request('GET', endpoint); return { experiments: (response.experiments || []).map((exp) => ({ experimentId: exp.experiment_id, experimentKey: exp.experiment_key, name: exp.name, status: exp.status, variantCount: exp.variant_count, totalEvaluations: exp.total_evaluations, averageScore: exp.average_score, totalCostUsd: exp.total_cost_usd, createdAt: exp.created_at })), total: response.total || 0 }; } /** * Get details for a specific prompt experiment by key */ async getPromptExperiment(experimentKey) { const response = await this.request('GET', `/internal/sdk/prompts/experiments/${experimentKey}`); return { experimentId: response.experiment_id, experimentKey: response.experiment_key, name: response.name, description: response.description, status: response.status, basePrompt: response.base_prompt, targetMetrics: response.target_metrics || [], configuration: response.configuration || {}, variants: (response.variants || []).map((v) => ({ variantId: v.variant_id, name: v.name, promptTemplate: v.prompt_template, isControl: v.is_control, trafficAllocation: v.traffic_allocation })), createdAt: response.created_at, updatedAt: response.updated_at }; } /** * Get details for a specific prompt experiment by ID */ async getPromptExperimentById(experimentId) { const response = await this.request('GET', `/internal/sdk/prompts/experiments/by-id/${experimentId}`); return { experimentId: response.experiment_id, experimentKey: response.experiment_key, name: response.name, description: response.description, status: response.status, basePrompt: response.base_prompt, targetMetrics: response.target_metrics || [], configuration: response.configuration || {}, variants: (response.variants || []).map((v) => ({ variantId: v.variant_id, name: v.name, promptTemplate: v.prompt_template, isControl: v.is_control, trafficAllocation: v.traffic_allocation })), createdAt: response.created_at, updatedAt: response.updated_at }; } // =========================================================================== // Metrics Tracking // =========================================================================== /** * Track an experiment metric */ async trackMetric(request) { await this.request('POST', '/internal/sdk/experiments/track-metric', { experiment_id: request.experimentId, user_id: request.userId, metric_key: request.metricKey, value: request.value, variant_id: request.variantId, session_id: request.sessionId, metadata: request.metadata, timestamp: request.timestamp || new Date().toISOString() }); return { success: true }; } /** * Track a single event */ async trackEvent(request) { await this.request('POST', '/internal/sdk/events', { event_key: request.eventKey, user_id: request.userId, event_type: request.eventType || 'metric_event', value: request.value, experiment_id: request.experimentId, variant_id: request.variantId, session_id: request.sessionId, metadata: request.metadata, timestamp: request.timestamp || new Date().toISOString() }); return { success: true }; } /** * Track multiple events in batch */ async trackEventBatch(events) { const formattedEvents = events.map(e => ({ event_key: e.eventKey, user_id: e.userId, event_type: e.eventType || 'metric_event', value: e.value, experiment_id: e.experimentId, variant_id: e.variantId, session_id: e.sessionId, metadata: e.metadata, timestamp: e.timestamp || new Date().toISOString() })); await this.request('POST', '/internal/sdk/events/batch', { events: formattedEvents }); return { success: true, count: events.length }; } /** * Get success metrics for an experiment */ async getExperimentSuccessMetrics(experimentId) { const response = await this.request('GET', `/sdk/v1/experiments/${experimentId}/success-metrics`); return (response.metrics || response || []).map((m) => ({ metricKey: m.metric_key || m.metricKey, displayName: m.display_name || m.displayName, calculationMethod: m.calculation_method || m.calculationMethod, isPrimary: m.is_primary || m.isPrimary })); } // =========================================================================== // Async Scoring (41D Quality Evaluation) // =========================================================================== /** * Submit a scoring request for async processing * * This submits a prompt-response pair for quality evaluation using * Variably's 41-dimension scoring system. The scoring is performed * asynchronously - use getScoringRequest() to poll for results. * * @param request - The scoring request with prompt/response * @returns Submit response with request ID for polling */ async submitScoringRequest(request) { const response = await this.request('POST', '/scoring/requests', { experiment_id: request.experimentId, variant_id: request.variantId, project_id: request.projectId, prompt: request.prompt, response: request.response, provider: request.provider, model: request.model, temperature: request.temperature, latency_ms: request.latencyMs, prompt_tokens: request.promptTokens, output_tokens: request.outputTokens, total_tokens: request.totalTokens, source: request.source || 'cloud', agent_id: request.agentId, evaluation_context: request.evaluationContext ? this.convertEvaluationContext(request.evaluationContext) : undefined }); return { requestId: response.request_id, status: response.status, message: response.message, pollEndpoint: response.poll_endpoint }; } /** * Get a scoring request by ID (poll for results) * * @param requestId - The scoring request ID * @returns The scoring request with status and results (if completed) */ async getScoringRequest(requestId) { const response = await this.request('GET', `/scoring/requests/${requestId}`); return this.convertScoringRequest(response); } /** * Get scoring requests for an experiment * * @param experimentId - The experiment ID * @param options - Pagination options * @returns List of scoring requests */ async getScoringRequestsByExperiment(experimentId, options) { const params = new URLSearchParams(); if (options?.limit) params.set('limit', String(options.limit)); if (options?.offset) params.set('offset', String(options.offset)); const endpoint = `/scoring/experiments/${experimentId}/requests${params.toString() ? `?${params}` : ''}`; const response = await this.request('GET', endpoint); return { requests: (response.requests || []).map((r) => this.convertScoringRequest(r)), limit: response.limit || 50, offset: response.offset || 0 }; } /** * Get scoring statistics for a project * * @param projectId - The project ID * @returns Scoring statistics */ async getScoringStats(projectId) { const response = await this.request('GET', `/scoring/stats/${projectId}`); return { totalRequests: response.total_requests || 0, pendingRequests: response.pending_requests || 0, completedRequests: response.completed_requests || 0, failedRequests: response.failed_requests || 0, avgProcessingMs: response.avg_processing_ms || 0, avgOverallScore: response.avg_overall_score || 0 }; } /** * Submit scoring request and wait for results * * This is a convenience method that submits a scoring request and * polls until the result is ready or timeout is reached. * * @param request - The scoring request * @param options - Polling options * @returns The completed scoring result */ async submitAndWaitForScoring(request, options) { const timeout = options?.timeoutMs || 60000; const pollInterval = options?.pollIntervalMs || 1000; const startTime = Date.now(); // Submit the request const submitResponse = await this.submitScoringRequest(request); const requestId = submitResponse.requestId; // Poll for results while (Date.now() - startTime < timeout) { const scoringRequest = await this.getScoringRequest(requestId); if (scoringRequest.status === 'completed' || scoringRequest.status === 'failed') { return scoringRequest; } // Wait before next poll await new Promise(resolve => setTimeout(resolve, pollInterval)); } // Timeout - return last known state return this.getScoringRequest(requestId); } /** * Convert raw scoring request response to typed object */ convertScoringRequest(response) { return { id: response.id, requestId: response.request_id, experimentId: response.experiment_id, variantId: response.variant_id, projectId: response.project_id, status: response.status, source: response.source, agentId: response.agent_id, result: response.result ? this.convertScoringResult(response.result) : undefined, error: response.error, createdAt: response.created_at, updatedAt: response.updated_at, processedAt: response.processed_at }; } /** * Convert raw scoring result to typed object */ convertScoringResult(result) { return { requestId: result.request_id, experimentId: result.experiment_id, variantId: result.variant_id, projectId: result.project_id, overallScore: result.overall_score, qualityScore: result.quality_score, safetyScore: result.safety_score, semanticScore: result.semantic_score, advancedScore: result.advanced_score, dimensionScores: result.dimension_scores || {}, metadata: result.metadata ? { prompt: { charCount: result.metadata.prompt?.char_count || 0, wordCount: result.metadata.prompt?.word_count || 0, sentenceCount: result.metadata.prompt?.sentence_count || 0, questionCount: result.metadata.prompt?.question_count || 0, avgWordLength: result.metadata.prompt?.avg_word_length || 0, avgSentenceLength: result.metadata.prompt?.avg_sentence_length || 0, readabilityScore: result.metadata.prompt?.readability_score || 0, hasQuestion: result.metadata.prompt?.has_question || false, hasInstruction: result.metadata.prompt?.has_instruction || false, hasCodeRequest: result.metadata.prompt?.has_code_request || false }, response: { charCount: result.metadata.response?.char_count || 0, wordCount: result.metadata.response?.word_count || 0, sentenceCount: result.metadata.response?.sentence_count || 0, hasCode: result.metadata.response?.has_code || false, codeBlockCount: result.metadata.response?.code_block_count || 0, avgWordLength: result.metadata.response?.avg_word_length || 0, avgSentenceLength: result.metadata.response?.avg_sentence_length || 0, readabilityScore: result.metadata.response?.readability_score || 0, vocabularyRichness: result.metadata.response?.vocabulary_richness || 0, containsPII: result.metadata.response?.contains_pii || false, toxicityScore: result.metadata.response?.toxicity_score || 0, sentimentScore: result.metadata.response?.sentiment_score || 0, sentimentLabel: result.metadata.response?.sentiment_label || 'neutral' }, comparative: { responsePromptRatio: result.metadata.comparative?.response_prompt_ratio || 0, keywordOverlap: result.metadata.comparative?.keyword_overlap || 0, topicAlignment: result.metadata.comparative?.topic_alignment || 0, formatCompliance: result.metadata.comparative?.format_compliance || 0 } } : undefined, processedAt: result.processed_at, processingMs: result.processing_ms || 0, error: result.error }; } // =========================================================================== // Streaming Prompt Evaluation (SSE) // =========================================================================== /** * Evaluate a prompt experiment with streaming via Server-Sent Events. * * Connects to the SSE endpoint and invokes callbacks as tokens arrive. * Use this for real-time token-by-token display in chat UIs. * * @param experimentKey - Experiment key * @param inputVariables - Template variables * @param userContext - User context for variant selection * @param callbacks - Streaming callbacks * @param options - Evaluation options * @returns AbortController to cancel the stream */ evaluatePromptStream(experimentKey, inputVariables, userContext, callbacks, options = {}) { const controller = new AbortController(); const url = `${this.config.baseUrl}/api/v1/internal/sdk/prompt-experiments/evaluate-stream`; const body = JSON.stringify({ experiment_key: experimentKey, input_variables: inputVariables, context: this.convertUserContext(userContext), evaluation_context: options.evaluationContext ? this.convertEvaluationContext(options.evaluationContext) : undefined, }); // Run the streaming fetch in the background (async () => { try { const response = await fetch(url, { method: 'POST', headers: { 'X-API-Key': this.config.apiKey, 'Content-Type': 'application/json', 'Accept': 'text/event-stream', }, body, signal: controller.signal, }); if (!response.ok) { const errorData = await response.json().catch(() => ({})); throw new NetworkError(`HTTP ${response.status}: ${errorData.error || response.statusText}`, response.status, url); } if (!response.body) { throw new NetworkError('No response body for SSE stream', 0, url); } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ''; let currentEventType = ''; while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split('\n'); buffer = lines.pop() || ''; // Keep incomplete last line in buffer for (const line of lines) { if (line.startsWith('event: ')) { currentEventType = line.slice(7).trim(); } else if (line.startsWith('data: ')) { const data = line.slice(6); try { const parsed = JSON.parse(data); switch (currentEventType) { case 'token': callbacks.onToken(parsed.content || ''); break; case 'variant': callbacks.onVariant?.({ experimentId: parsed.experiment_id, variantId: parsed.variant_id, variantKey: parsed.variant_key, model: parsed.model, provider: parsed.provider, }); break; case 'metadata': callbacks.onMetadata?.({ executionId: parsed.execution_id, tokenUsage: { promptTokens: parsed.token_usage?.prompt_tokens || 0, completionTokens: parsed.token_usage?.completion_tokens || 0, totalTokens: parsed.token_usage?.total_tokens || 0, }, latencyMs: parsed.latency_ms || 0, qualityScore: parsed.quality_score, }); break; case 'done': callbacks.onComplete?.(); break; case 'error': callbacks.onError?.(new Error(parsed.message || 'Stream error')); break; } } catch { // Non-JSON data line, skip } currentEventType = ''; } } } // Stream ended without explicit 'done' event callbacks.onComplete?.(); } catch (error) { if (error?.name === 'AbortError') return; callbacks.onError?.(error instanceof Error ? error : new Error(String(error))); } })(); return controller; } // =========================================================================== // Variant Assignment // =========================================================================== /** * Assign a variant to a user for an experiment */ async assignVariant(experimentId, userContext) { const response = await this.request('POST', `/internal/sdk/experiments/${experimentId}/assign`, { context: this.convertUserContext(userContext) }); return { experimentId: response.experiment_id, experimentKey: response.experiment_key, variant: { variantId: response.variant.variant_id, name: response.variant.name, promptTemplate: response.variant.prompt_template || '', isControl: response.variant.is_control, trafficAllocation: response.variant.traffic_allocation }, reason: response.reason, timestamp: response.timestamp }; } // =========================================================================== // Feature Flags and Gates (REST variants) // =========================================================================== /** * Evaluate a feature flag via REST API */ async evaluateFlag(flagKey, userContext) { const response = await this.request('POST', '/internal/sdk/evaluate', { flag_key: flagKey, context: this.convertUserContext(userContext) }); return { flagKey: response.flag_key, value: response.value, reason: response.reason, ruleId: response.rule_id, experimentId: response.experiment_id, variantId: response.variant_id }; } /** * Evaluate a feature gate via REST API */ async evaluateGate(gateKey, userContext) { const response = await this.request('POST', '/internal/sdk/feature-gates/evaluate', { gate_key: gateKey, context: this.convertUserContext(userContext) }); return { gateKey: response.gate_key, value: response.value, reason: response.reason, ruleId: response.rule_id, experimentId: response.experiment_id, variantId: response.variant_id, successMetrics: response.success_metrics }; } // =========================================================================== // LLM Execution // =========================================================================== /** * Execute an LLM prompt directly */ async executeLLMPrompt(experimentKey, inputVariables, userContext, options) { const response = await this.request('POST', '/internal/sdk/llm/execute', { experiment_key: experimentKey, input_variables: inputVariables, context: this.convertUserContext(userContext), provider: options?.provider, model: options?.model, temperature: options?.temperature, max_tokens: options?.maxTokens }); return { response: response.response, variantId: response.variant_id, experimentId: response.experiment_id, usage: response.usage ? { promptTokens: response.usage.prompt_tokens, completionTokens: response.usage.completion_tokens, totalTokens: response.usage.total_tokens } : undefined, latencyMs: response.latency_ms }; } // =========================================================================== // Dynamic Configuration // =========================================================================== /** * Evaluate a dynamic config */ async evaluateConfig(configKey, userContext) { const response = await this.request('POST', '/sdk/v1/configs/evaluate', { config_key: configKey, context: this.convertUserContext(userContext) }); return { configKey: response.config_key, value: response.value, reason: response.reason, etag: response.etag, version: response.version }; } /** * Evaluate multiple dynamic configs in batch */ async evaluateConfigBatch(configKeys, userContext) { const response = await this.request('POST', '/sdk/v1/configs/evaluate/batch', { config_keys: configKeys, context: this.convertUserContext(userContext) }); const results = {}; for (const [key, result] of Object.entries(response.results || {})) { const r = result; results[key] = { configKey: r.config_key, value: r.value, reason: r.reason, etag: r.etag, version: r.version }; } return results; } // =========================================================================== // Private Methods // =========================================================================== async request(method, endpoint, data) { const url = `${this.config.baseUrl}/api/v1${endpoint}`; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), this.config.timeout); let lastError = null; for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) { try { const options = { method, headers: { 'X-API-Key': this.config.apiKey, 'Content-Type': 'application/json', }, signal: controller.signal, }; if (data) { options.body = JSON.stringify(data); } this.logger.debug('Making request', { method, url, attempt }); const response = await fetch(url, options); clearTimeout(timeoutId); if (!response.ok) { const errorData = await response.json().catch(() => ({})); if (response.status === 401) { throw new AuthenticationError('Invalid or missing API key'); } if (response.status === 400) { throw new ValidationError(errorData.error || errorData.message || 'Validation error', errorData.field || 'request'); } throw new NetworkError(`HTTP ${response.status}: ${errorData.error || errorData.message || response.statusText}`, response.status, url); } const result = await response.json(); this.logger.debug('Request successful', { method, url }); return result; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); // Don't retry on auth or validation errors if (error instanceof AuthenticationError || error instanceof ValidationError) { throw error; } // Retry on network errors if (attempt < this.config.retryAttempts - 1) { const delay = Math.pow(2, attempt) * 100; // Exponential backoff this.logger.warn('Request failed, retrying', { attempt, delay, error: lastError.message }); await new Promise(resolve => setTimeout(resolve, delay)); } } } clearTimeout(timeoutId); throw lastError || new NetworkError('Request failed after retries', 0, url); } convertEvaluationContext(ctx) { return { reference_materials: ctx.referenceMaterials?.map(ref => ({ id: ref.id, content: ref.content, source: ref.source, type: ref.type, relevance_score: ref.relevanceScore, })), workflow_history: ctx.workflowHistory?.map(step => ({ role: step.role, step: step.step, input: step.input, output: step.output, content: step.content, })), retrieval_query: ctx.retrievalQuery, }; } convertUserContext(context) { return { user_id: context.userId, email: context.email, country: context.country, language: context.language, platform: context.platform, version: context.version, ip_address: context.ipAddress, user_agent: context.userAgent, session_id: context.sessionId, attributes: context.attributes }; } convertPromptEvaluationResponse(response) { return { experimentId: response.experiment_id, experimentKey: response.experiment_key, variantId: response.variant_id || response.variant_used, variantName: response.variant_name || response.variant_used, isControl: response.is_control || false, response: response.response, prompt: response.prompt || '', metadata: response.metadata || {}, performance: response.performance ? { overallScore: response.performance.overall_score, metricsScores: response.performance.metrics_scores || {}, latencyMs: response.performance.latency_ms, tokenCount: response.performance.token_count, promptTokens: response.performance.prompt_tokens, completionTokens: response.performance.completion_tokens } : undefined, cost: response.cost ? { costUsd: response.cost.cost_usd, providerName: response.cost.provider_name, modelName: response.cost.model_name } : undefined, timestamp: response.timestamp || new Date().toISOString() }; } // =========================================================================== // Observe Mode — Zero-friction logging // =========================================================================== /** * Log a prompt/response pair for 43-dimension evaluation (Observe Mode). * * No experiment creation needed. Just log your prompts and responses * and see quality scores in the dashboard. * * @example * ```ts * const result = await client.log({ * prompt: "What is TypeScript?", * response: "TypeScript is a typed superset of JavaScript.", * provider: "openai", * model: "gpt-4", * }); * console.log(result.observationId); * ``` */ async log(params) { const body = { prompt: params.prompt, response: params.response, }; // Build evaluation_context from multi-turn/RAG params const evaluationContext = {}; if (params.conversationHistory?.length) { evaluationContext.conversation_history = params.conversationHistory.map(m => ({ role: m.role, content: m.content, })); } if (params.referenceMaterials?.length) { evaluationContext.reference_materials = params.referenceMaterials.map(r => ({ id: r.id, content: r.content, source: r.source, type: r.type, relevance_score: r.relevanceScore, })); } if (params.retrievalQuery) { evaluationContext.retrieval_query = params.retrievalQuery; } if (Object.keys(evaluationContext).length > 0) { body.evaluation_context = evaluationContext; } if (params.tags) body.tags = params.tags; if (params.userId) body.user_id = params.userId; if (params.sessionId) body.session_id = params.sessionId; if (params.provider) body.provider = params.provider; if (params.model) body.model = params.model; if (params.latencyMs !== undefined) body.latency_ms = params.latencyMs; if (params.promptTokens !== undefined) body.prompt_tokens = params.promptTokens; if (params.completionTokens !== undefined) body.completion_tokens = params.completionTokens; if (params.metadata) body.metadata = params.metadata; const response = await this.request('POST', '/internal/sdk/observe/log', body); return { observationId: response.observation_id, experimentId: response.experiment_id, status: response.status, }; } normalizeConfig(config) { if (!config.apiKey) { throw new ValidationError('API key is required', 'apiKey'); } return { apiKey: config.apiKey, baseUrl: config.baseUrl || 'https://api.variably.io', timeout: config.timeout || 30000, retryAttempts: config.retryAttempts || 3, debug: config.debug || false }; } } // ============================================================================= // Factory Functions // ============================================================================= /** * Create a new REST client */ export function createRESTClient(config) { return new VariablyRESTClient(config); } /** * Create a REST client from environment variables */ export function createRESTClientFromEnv() { return new VariablyRESTClient({ apiKey: process.env.VARIABLY_API_KEY || '', baseUrl: process.env.VARIABLY_BASE_URL, timeout: process.env.VARIABLY_TIMEOUT ? parseInt(process.env.VARIABLY_TIMEOUT, 10) : undefined, debug: process.env.VARIABLY_DEBUG === 'true' }); } // ============================================================================= // Prompt Context Builder (Helper) // ============================================================================= /** * Builder class for constructing prompt input variables */ export class PromptContextBuilder { constructor() { this.variables = {}; } /** * Set product information */ product(name, price, features) { this.variables.product_name = name; if (price) this.variables.price = price; if (features) this.variables.features = features.join(', '); return this; } /** * Set customer information */ customer(name, segment, preferences) { if (name) this.variables.customer_name = name; if (segment) this.variables.customer_segment = segment; if (preferences) this.variables.customer_preferences = preferences.join(', '); return this; } /** * Set tone for the prompt */ tone(tone) { this.variables.tone = tone; return this; } /** * Set target audience */ audience(audience) { this.variables.target_audience = audience; return this; } /** * Set a custom variable */ custom(key, value) { this.variables[key] = value; return this; } /** * Build the variables object */ build() { return { ...this.variables }; } } //# sourceMappingURL=rest-client.js.map