@latitude-data/sdk/dist/index.d.ts

Latitude SDK for Typescript

import { z } from 'zod';
import { FinishReason, ToolResultPart, TextStreamPart } from 'ai';
import { JSONSchema7 } from 'json-schema';
import { AdapterMessageType, Message as Message$1, ProviderAdapter, Config as Config$1, ToolCallContent } from 'promptl-ai';
export { Adapters, ContentType } from 'promptl-ai';

declare enum ParameterType {
    Text = "text",
    Image = "image",
    File = "file"
}
declare enum LatitudeTool {
    RunCode = "code",
    WebSearch = "search",
    WebExtract = "extract",
    Think = "think",
    TODO = "todo"
}

declare const CompositeEvaluationAverageSpecification: {
    readonly name: "Average";
    readonly description: "Combines scores evenly. The resulting score is the average";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        evaluationUuids: z.ZodArray<z.ZodString>;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
        defaultTarget: z.ZodOptional<z.ZodBoolean>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        results: z.ZodRecord<z.ZodString, z.ZodObject<{
            uuid: z.ZodString;
            name: z.ZodString;
            score: z.ZodNumber;
            reason: z.ZodString;
            passed: z.ZodBoolean;
        }, z.core.$strip>>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            evaluationUuids: z.ZodArray<z.ZodString>;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
            defaultTarget: z.ZodOptional<z.ZodBoolean>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        errors: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
            uuid: z.ZodString;
            name: z.ZodString;
            message: z.ZodString;
        }, z.core.$strip>>>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Composite, CompositeEvaluationMetric.Average>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type CompositeEvaluationAverageResultMetadata = z.infer<typeof CompositeEvaluationAverageSpecification.resultMetadata>;
type CompositeEvaluationAverageResultError = z.infer<typeof CompositeEvaluationAverageSpecification.resultError>;
declare const CompositeEvaluationWeightedSpecification: {
    readonly name: "Weighted";
    readonly description: "Combines scores using custom weights. The resulting score is the weighted blend";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        evaluationUuids: z.ZodArray<z.ZodString>;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
        defaultTarget: z.ZodOptional<z.ZodBoolean>;
        weights: z.ZodRecord<z.ZodString, z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        results: z.ZodRecord<z.ZodString, z.ZodObject<{
            uuid: z.ZodString;
            name: z.ZodString;
            score: z.ZodNumber;
            reason: z.ZodString;
            passed: z.ZodBoolean;
        }, z.core.$strip>>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            evaluationUuids: z.ZodArray<z.ZodString>;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
            defaultTarget: z.ZodOptional<z.ZodBoolean>;
            weights: z.ZodRecord<z.ZodString, z.ZodNumber>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        errors: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
            uuid: z.ZodString;
            name: z.ZodString;
            message: z.ZodString;
        }, z.core.$strip>>>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Composite, CompositeEvaluationMetric.Weighted>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type CompositeEvaluationWeightedResultMetadata = z.infer<typeof CompositeEvaluationWeightedSpecification.resultMetadata>;
type CompositeEvaluationWeightedResultError = z.infer<typeof CompositeEvaluationWeightedSpecification.resultError>;
declare const CompositeEvaluationCustomSpecification: {
    readonly name: "Custom";
    readonly description: "Combines scores using a custom formula. The resulting score is the result of the expression";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        evaluationUuids: z.ZodArray<z.ZodString>;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
        defaultTarget: z.ZodOptional<z.ZodBoolean>;
        formula: z.ZodString;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        results: z.ZodRecord<z.ZodString, z.ZodObject<{
            uuid: z.ZodString;
            name: z.ZodString;
            score: z.ZodNumber;
            reason: z.ZodString;
            passed: z.ZodBoolean;
        }, z.core.$strip>>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            evaluationUuids: z.ZodArray<z.ZodString>;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
            defaultTarget: z.ZodOptional<z.ZodBoolean>;
            formula: z.ZodString;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        errors: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
            uuid: z.ZodString;
            name: z.ZodString;
            message: z.ZodString;
        }, z.core.$strip>>>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Composite, CompositeEvaluationMetric.Custom>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type CompositeEvaluationCustomResultMetadata = z.infer<typeof CompositeEvaluationCustomSpecification.resultMetadata>;
type CompositeEvaluationCustomResultError = z.infer<typeof CompositeEvaluationCustomSpecification.resultError>;
declare enum CompositeEvaluationMetric {
    Average = "average",
    Weighted = "weighted",
    Custom = "custom"
}
type CompositeEvaluationResultMetadata<M extends CompositeEvaluationMetric = CompositeEvaluationMetric> = M extends CompositeEvaluationMetric.Average ? CompositeEvaluationAverageResultMetadata : M extends CompositeEvaluationMetric.Weighted ? CompositeEvaluationWeightedResultMetadata : M extends CompositeEvaluationMetric.Custom ? CompositeEvaluationCustomResultMetadata : never;
type CompositeEvaluationResultError<M extends CompositeEvaluationMetric = CompositeEvaluationMetric> = M extends CompositeEvaluationMetric.Average ? CompositeEvaluationAverageResultError : M extends CompositeEvaluationMetric.Weighted ? CompositeEvaluationWeightedResultError : M extends CompositeEvaluationMetric.Custom ? CompositeEvaluationCustomResultError : never;

declare const HumanEvaluationBinarySpecification: {
    readonly name: "Binary";
    readonly description: "Judges whether the response meets the criteria. The resulting score is \"passed\" or \"failed\"";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        enableControls: z.ZodOptional<z.ZodBoolean>;
        criteria: z.ZodOptional<z.ZodString>;
        passDescription: z.ZodOptional<z.ZodString>;
        failDescription: z.ZodOptional<z.ZodString>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        reason: z.ZodOptional<z.ZodString>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            enableControls: z.ZodOptional<z.ZodBoolean>;
            criteria: z.ZodOptional<z.ZodString>;
            passDescription: z.ZodOptional<z.ZodString>;
            failDescription: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Human, HumanEvaluationMetric.Binary>) => string | undefined;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: false;
    readonly supportsManualEvaluation: true;
};
type HumanEvaluationBinaryResultMetadata = z.infer<typeof HumanEvaluationBinarySpecification.resultMetadata>;
type HumanEvaluationBinaryResultError = z.infer<typeof HumanEvaluationBinarySpecification.resultError>;
declare const HumanEvaluationRatingSpecification: {
    readonly name: "Rating";
    readonly description: "Judges the response by rating it under a criteria. The resulting score is the rating";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        enableControls: z.ZodOptional<z.ZodBoolean>;
        criteria: z.ZodOptional<z.ZodString>;
        minRating: z.ZodNumber;
        minRatingDescription: z.ZodOptional<z.ZodString>;
        maxRating: z.ZodNumber;
        maxRatingDescription: z.ZodOptional<z.ZodString>;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        reason: z.ZodOptional<z.ZodString>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            enableControls: z.ZodOptional<z.ZodBoolean>;
            criteria: z.ZodOptional<z.ZodString>;
            minRating: z.ZodNumber;
            minRatingDescription: z.ZodOptional<z.ZodString>;
            maxRating: z.ZodNumber;
            maxRatingDescription: z.ZodOptional<z.ZodString>;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Human, HumanEvaluationMetric.Rating>) => string | undefined;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: false;
    readonly supportsManualEvaluation: true;
};
type HumanEvaluationRatingResultMetadata = z.infer<typeof HumanEvaluationRatingSpecification.resultMetadata>;
type HumanEvaluationRatingResultError = z.infer<typeof HumanEvaluationRatingSpecification.resultError>;
declare enum HumanEvaluationMetric {
    Binary = "binary",
    Rating = "rating"
}
type HumanEvaluationResultMetadata<M extends HumanEvaluationMetric = HumanEvaluationMetric> = M extends HumanEvaluationMetric.Binary ? HumanEvaluationBinaryResultMetadata : M extends HumanEvaluationMetric.Rating ? HumanEvaluationRatingResultMetadata : never;
type HumanEvaluationResultError<M extends HumanEvaluationMetric = HumanEvaluationMetric> = M extends HumanEvaluationMetric.Binary ? HumanEvaluationBinaryResultError : M extends HumanEvaluationMetric.Rating ? HumanEvaluationRatingResultError : never;

declare const LlmEvaluationBinarySpecification: {
    readonly name: "Binary";
    readonly description: "Judges whether the response meets the criteria. The resulting score is \"passed\" or \"failed\"";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        provider: z.ZodString;
        model: z.ZodString;
        criteria: z.ZodString;
        passDescription: z.ZodString;
        failDescription: z.ZodString;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        evaluationLogId: z.ZodNumber;
        reason: z.ZodString;
        tokens: z.ZodNumber;
        cost: z.ZodNumber;
        duration: z.ZodNumber;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            provider: z.ZodString;
            model: z.ZodString;
            criteria: z.ZodString;
            passDescription: z.ZodString;
            failDescription: z.ZodString;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        runErrorId: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Llm, LlmEvaluationMetric.Binary>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: true;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type LlmEvaluationBinaryResultMetadata = z.infer<typeof LlmEvaluationBinarySpecification.resultMetadata>;
type LlmEvaluationBinaryResultError = z.infer<typeof LlmEvaluationBinarySpecification.resultError>;
declare const LlmEvaluationRatingSpecification: {
    readonly name: "Rating";
    readonly description: "Judges the response by rating it under a criteria. The resulting score is the rating";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        provider: z.ZodString;
        model: z.ZodString;
        criteria: z.ZodString;
        minRating: z.ZodNumber;
        minRatingDescription: z.ZodString;
        maxRating: z.ZodNumber;
        maxRatingDescription: z.ZodString;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        evaluationLogId: z.ZodNumber;
        reason: z.ZodString;
        tokens: z.ZodNumber;
        cost: z.ZodNumber;
        duration: z.ZodNumber;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            provider: z.ZodString;
            model: z.ZodString;
            criteria: z.ZodString;
            minRating: z.ZodNumber;
            minRatingDescription: z.ZodString;
            maxRating: z.ZodNumber;
            maxRatingDescription: z.ZodString;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        runErrorId: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Llm, LlmEvaluationMetric.Rating>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: true;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type LlmEvaluationRatingResultMetadata = z.infer<typeof LlmEvaluationRatingSpecification.resultMetadata>;
type LlmEvaluationRatingResultError = z.infer<typeof LlmEvaluationRatingSpecification.resultError>;
declare const LlmEvaluationComparisonSpecification: {
    readonly name: "Comparison";
    readonly description: "Judges the response by comparing the criteria to the expected output. The resulting score is the percentage of compared criteria that is met";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        provider: z.ZodString;
        model: z.ZodString;
        criteria: z.ZodString;
        passDescription: z.ZodString;
        failDescription: z.ZodString;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        evaluationLogId: z.ZodNumber;
        reason: z.ZodString;
        tokens: z.ZodNumber;
        cost: z.ZodNumber;
        duration: z.ZodNumber;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            provider: z.ZodString;
            model: z.ZodString;
            criteria: z.ZodString;
            passDescription: z.ZodString;
            failDescription: z.ZodString;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        runErrorId: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Llm, LlmEvaluationMetric.Comparison>) => string;
    readonly requiresExpectedOutput: true;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type LlmEvaluationComparisonResultMetadata = z.infer<typeof LlmEvaluationComparisonSpecification.resultMetadata>;
type LlmEvaluationComparisonResultError = z.infer<typeof LlmEvaluationComparisonSpecification.resultError>;
declare const LlmEvaluationCustomSpecification: {
    readonly name: "Custom";
    readonly description: "Judges the response under a criteria using a custom prompt. The resulting score is the value of criteria that is met";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        provider: z.ZodString;
        model: z.ZodString;
        prompt: z.ZodString;
        minScore: z.ZodNumber;
        maxScore: z.ZodNumber;
        minThreshold: z.ZodOptional<z.ZodNumber>;
        maxThreshold: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        evaluationLogId: z.ZodNumber;
        reason: z.ZodString;
        tokens: z.ZodNumber;
        cost: z.ZodNumber;
        duration: z.ZodNumber;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            provider: z.ZodString;
            model: z.ZodString;
            prompt: z.ZodString;
            minScore: z.ZodNumber;
            maxScore: z.ZodNumber;
            minThreshold: z.ZodOptional<z.ZodNumber>;
            maxThreshold: z.ZodOptional<z.ZodNumber>;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
        runErrorId: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Llm, LlmEvaluationMetric.Custom>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: true;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type LlmEvaluationCustomResultMetadata = z.infer<typeof LlmEvaluationCustomSpecification.resultMetadata>;
type LlmEvaluationCustomResultError = z.infer<typeof LlmEvaluationCustomSpecification.resultError>;
declare enum LlmEvaluationMetric {
    Binary = "binary",
    Rating = "rating",
    Comparison = "comparison",
    Custom = "custom",
    CustomLabeled = "custom_labeled"
}
type LlmEvaluationResultMetadata<M extends LlmEvaluationMetric = LlmEvaluationMetric> = M extends LlmEvaluationMetric.Binary ? LlmEvaluationBinaryResultMetadata : M extends LlmEvaluationMetric.Rating ? LlmEvaluationRatingResultMetadata : M extends LlmEvaluationMetric.Comparison ? LlmEvaluationComparisonResultMetadata : M extends LlmEvaluationMetric.Custom ? LlmEvaluationCustomResultMetadata : M extends LlmEvaluationMetric.CustomLabeled ? LlmEvaluationCustomResultMetadata : never;
type LlmEvaluationResultError<M extends LlmEvaluationMetric = LlmEvaluationMetric> = M extends LlmEvaluationMetric.Binary ? LlmEvaluationBinaryResultError : M extends LlmEvaluationMetric.Rating ? LlmEvaluationRatingResultError : M extends LlmEvaluationMetric.Comparison ? LlmEvaluationComparisonResultError : M extends LlmEvaluationMetric.Custom ? LlmEvaluationCustomResultError : M extends LlmEvaluationMetric.CustomLabeled ? LlmEvaluationCustomResultError : never;

declare const RuleEvaluationExactMatchSpecification: {
    readonly name: "Exact Match";
    readonly description: "Checks if the response is exactly the same as the expected output. The resulting score is \"matched\" or \"unmatched\"";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        caseInsensitive: z.ZodBoolean;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        reason: z.ZodOptional<z.ZodString>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            caseInsensitive: z.ZodBoolean;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Rule, RuleEvaluationMetric.ExactMatch>) => string;
    readonly requiresExpectedOutput: true;
    readonly supportsLiveEvaluation: false;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type RuleEvaluationExactMatchResultMetadata = z.infer<typeof RuleEvaluationExactMatchSpecification.resultMetadata>;
type RuleEvaluationExactMatchResultError = z.infer<typeof RuleEvaluationExactMatchSpecification.resultError>;
declare const RuleEvaluationRegularExpressionSpecification: {
    readonly name: "Regular Expression";
    readonly description: "Checks if the response matches the regular expression. The resulting score is \"matched\" or \"unmatched\"";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        pattern: z.ZodString;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        reason: z.ZodOptional<z.ZodString>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            pattern: z.ZodString;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Rule, RuleEvaluationMetric.RegularExpression>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: true;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type RuleEvaluationRegularExpressionResultMetadata = z.infer<typeof RuleEvaluationRegularExpressionSpecification.resultMetadata>;
type RuleEvaluationRegularExpressionResultError = z.infer<typeof RuleEvaluationRegularExpressionSpecification.resultError>;
declare const RuleEvaluationSchemaValidationSpecification: {
    readonly name: "Schema Validation";
    readonly description: "Checks if the response follows the schema. The resulting score is \"valid\" or \"invalid\"";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        format: z.ZodEnum<{
            json: "json";
        }>;
        schema: z.ZodString;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        reason: z.ZodOptional<z.ZodString>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            format: z.ZodEnum<{
                json: "json";
            }>;
            schema: z.ZodString;
        }, z.core.$strip>;
    }, z.core.$strip>;
    readonly resultError: z.ZodObject<{
        message: z.ZodString;
    }, z.core.$strip>;
    readonly resultReason: (result: EvaluationResultSuccessValue<EvaluationType.Rule, RuleEvaluationMetric.SchemaValidation>) => string;
    readonly requiresExpectedOutput: false;
    readonly supportsLiveEvaluation: true;
    readonly supportsBatchEvaluation: true;
    readonly supportsManualEvaluation: false;
};
type RuleEvaluationSchemaValidationResultMetadata = z.infer<typeof RuleEvaluationSchemaValidationSpecification.resultMetadata>;
type RuleEvaluationSchemaValidationResultError = z.infer<typeof RuleEvaluationSchemaValidationSpecification.resultError>;
declare const RuleEvaluationLengthCountSpecification: {
    readonly name: "Length Count";
    readonly description: "Checks if the response is of a certain length. The resulting score is the length of the response";
    readonly configuration: z.ZodObject<{
        reverseScale: z.ZodBoolean;
        actualOutput: z.ZodObject<{
            messageSelection: z.ZodEnum<{
                last: "last";
                all: "all";
            }>;
            contentFilter: z.ZodOptional<z.ZodEnum<{
                text: "text";
                file: "file";
                reasoning: "reasoning";
                image: "image";
                tool_call: "tool_call";
            }>>;
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>;
        expectedOutput: z.ZodOptional<z.ZodObject<{
            parsingFormat: z.ZodEnum<{
                string: "string";
                json: "json";
            }>;
            fieldAccessor: z.ZodOptional<z.ZodString>;
        }, z.core.$strip>>;
        algorithm: z.ZodEnum<{
            character: "character";
            word: "word";
            sentence: "sentence";
        }>;
        minLength: z.ZodOptional<z.ZodNumber>;
        maxLength: z.ZodOptional<z.ZodNumber>;
    }, z.core.$strip>;
    readonly resultMetadata: z.ZodObject<{
        actualOutput: z.ZodString;
        expectedOutput: z.ZodOptional<z.ZodString>;
        datasetLabel: z.ZodOptional<z.ZodString>;
        reason: z.ZodOptional<z.ZodString>;
        configuration: z.ZodObject<{
            reverseScale: z.ZodBoolean;
            actualOutput: z.ZodObject<{
                messageSelection: z.ZodEnum<{
                    last: "last";
                    all: "all";
                }>;
                contentFilter: z.ZodOptional<z.ZodEnum<{
                    text: "text";
                    file: "file";
                    reasoning: "reasoning";
                    image: "image";
                    tool_call: "tool_call";
                }>>;
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>;
            expectedOutput: z.ZodOptional<z.ZodObject<{
                parsingFormat: z.ZodEnum<{
                    string: "string";
                    json: "json";
                }>;
                fieldAccessor: z.ZodOptional<z.ZodString>;
            }, z.core.$strip>>;
            algorithm: z.ZodEnum<{
                character: "character";