notdiamond
Version:
The official TypeScript library for the Notdiamond API
252 lines • 9.74 kB
JavaScript
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
import { APIResource } from "../core/resource.mjs";
import { path } from "../internal/utils/path.mjs";
export class PromptOptimization extends APIResource {
/**
* Retrieve the complete results of a prompt optimization run, including optimized
* prompts for all target models.
*
* This endpoint returns the optimized prompts and evaluation metrics for each
* target model in your optimization request. Call this endpoint after the
* optimization status is 'completed' to get your optimized prompts.
*
* **Response Structure:**
*
* - **origin_model**: Baseline performance of your original prompt on the origin
* model
* - Includes: system_prompt, user_message_template, score, evaluation metrics,
* cost
* - **target_models**: Array of results for each target model
* - Includes: optimized system_prompt, user_message_template, template_fields
* - pre_optimization_score: Performance before optimization
* - post_optimization_score: Performance after optimization
* - Evaluation metrics and cost information
*
* **Using Optimized Prompts:**
*
* 1. Extract the `system_prompt` and `user_message_template` from each target
* model result
* 2. Use `user_message_template_fields` to know which fields to substitute
* 3. Apply the optimized prompts when calling the respective target models
* 4. Compare pre/post optimization scores to see improvement
*
* **Status Handling:**
*
* - If optimization is still processing, target model results will have
* `result_status: "processing"`
* - Only completed target models will have system_prompt and template values
* - Failed target models will have `result_status: "failed"` with null values
*
* **Cost Information:**
*
* - Each model result includes cost in USD for the optimization process
* - Costs vary based on model pricing and number of evaluation examples
* - Typical range: $0.10 - $2.00 per target model
*
* **Best Practices:**
*
* 1. Wait for status 'completed' before calling this endpoint
* 2. Check result_status for each target model
* 3. Validate that post_optimization_score > pre_optimization_score
* 4. Save optimized prompts for production use
* 5. A/B test optimized prompts against originals in production
*
* @example
* ```ts
* const response =
* await client.promptOptimization.getOptimizationResults(
* 'optimization_run_id',
* );
* ```
*/
getOptimizationResults(optimizationRunID, options) {
return this._client.get(path `/v2/prompt/optimizeResults/${optimizationRunID}`, options);
}
/**
* Check the status of a prompt optimization run.
*
* Use this endpoint to poll the status of your optimization request. Processing is
* asynchronous, so you'll need to check periodically until the status indicates
* completion.
*
* **Status Values:**
*
* - `created`: Initial state, not yet processing
* - `queued`: Waiting for processing capacity (check queue_position)
* - `processing`: Currently optimizing prompts
* - `completed`: All target models have been processed successfully
* - `failed`: One or more target models failed to process
*
* **Polling Recommendations:**
*
* - Poll every 30-60 seconds during processing
* - Check queue_position if status is 'queued' to estimate wait time
* - Stop polling once status is 'completed' or 'failed'
* - Use GET /v2/prompt/optimizeResults to retrieve results after completion
*
* **Queue Position:**
*
* - Only present when status is 'queued'
* - Lower numbers mean earlier processing (position 1 is next)
* - Typical wait time: 1-5 minutes per position
*
* **Note:** This endpoint only returns status information. To get the actual
* optimized prompts and evaluation results, use GET /v2/prompt/optimizeResults
* once status is 'completed'.
*
* @example
* ```ts
* const response =
* await client.promptOptimization.getOptimziationStatus(
* 'optimization_run_id',
* );
* ```
*/
getOptimziationStatus(optimizationRunID, options) {
return this._client.get(path `/v2/prompt/optimizeStatus/${optimizationRunID}`, options);
}
/**
* Optimize your prompt from one LLM to work optimally across different target
* LLMs.
*
* This endpoint automatically optimizes your prompt (system prompt + user message
* template) to improve accuracy on your use case across various models. Each model
* has unique characteristics, and what works well for GPT-5 might not work as well
* for Claude or Gemini.
*
* **How Prompt Optimization Works:**
*
* 1. You provide your current prompt and optionally your current origin model
* 2. You specify the target models you want to optimize your prompt to
* 3. You provide evaluation examples (golden records) with expected answers
* 4. The system runs optimization to find the best prompt for each target model
* 5. You receive optimized prompts that perform well on your target models
*
* **Evaluation Metrics:** Choose either a standard metric or provide custom
* evaluation:
*
* - **Standard metrics**: LLMaaJ:Sem_Sim_1 (semantic similarity), JSON_Match
* - **Custom evaluation**: Provide evaluation_config with your own LLM judge,
* prompt, and cutoff
*
* **Dataset Requirements:**
*
* - Minimum 25 examples in train_goldens (more examples = better optimization)
* - **Prototype mode**: Set `prototype_mode: true` to use as few as 3 examples for
* prototyping
* - Recommended when you don't have enough data yet to build a proof-of-concept
* - Note: Performance may be degraded compared to standard mode (25+ examples)
* - Trade-off: Faster iteration with less data vs. potentially less
* generalizability
* - Each example must have fields matching your template placeholders
* - Supervised evaluation requires 'answer' field in each golden record
* - Unsupervised evaluation can work without answers
*
* **Training Time:**
*
* - Processing is asynchronous and typically takes 10-30 minutes
* - Time depends on: number of target models, dataset size, model availability
* - Use the returned optimization_run_id to check status and retrieve results
*
* **Example Workflow:**
*
* ```
* 1. POST /v2/prompt/optimize - Submit optimization request
* 2. GET /v2/prompt/optimizeStatus/{id} - Poll status until completed
* 3. GET /v2/prompt/optimizeResults/{id} - Retrieve optimized prompts
* 4. Use optimized prompts in production with target models
* ```
*
* @example
* ```ts
* const response = await client.promptOptimization.optimize({
* fields: ['question'],
* system_prompt: 'You are a mathematical assistant that counts digits accurately.',
* target_models: [
* { model: 'claude-sonnet-4-5-20250929', provider: 'anthropic' },
* { model: 'gemini-2.5-flash', provider: 'google' },
* ],
* template: 'Question: {question}\nAnswer:',
* evaluation_metric: 'LLMaaJ:Sem_Sim_1',
* prototype_mode: true,
* test_goldens: [
* {
* fields: { ... },
* answer: '15',
* },
* {
* fields: { ... },
* answer: '8',
* },
* {
* fields: { ... },
* answer: '1',
* },
* {
* fields: { ... },
* answer: '10',
* },
* {
* fields: { ... },
* answer: '11',
* },
* ],
* train_goldens: [
* {
* fields: { ... },
* answer: '20',
* },
* {
* fields: { ... },
* answer: '10',
* },
* {
* fields: { ... },
* answer: '0',
* },
* {
* fields: { ... },
* answer: '16',
* },
* {
* fields: { ... },
* answer: '2',
* },
* ],
* });
* ```
*/
optimize(body, options) {
return this._client.post('/v2/prompt/optimize', { body, ...options });
}
/**
* Get LLM usage costs for a specific prompt optimization run.
*
* This endpoint returns the total cost and detailed usage records for all LLM
* requests made during a prompt optimization run. Use this to track costs
* associated with optimizing prompts for different target models.
*
* **Cost Breakdown:**
*
* - Total cost across all models used in the optimization
* - Individual usage records with provider, model, tokens, and costs
* - Timestamps for each LLM request
*
* **Access Control:**
*
* - Only accessible by the user who created the optimization run
* - Requires prompt optimization access
*
* @example
* ```ts
* const response =
* await client.promptOptimization.retrieveCosts(
* 'optimization_run_id',
* );
* ```
*/
retrieveCosts(optimizationRunID, options) {
return this._client.get(path `/v2/prompt/optimize/${optimizationRunID}/costs`, options);
}
}
//# sourceMappingURL=prompt-optimization.mjs.map