@arizeai/phoenix-evals/dist/src/llm/createToolInvocationEvaluator.d.ts

A library for running evaluations for AI use cases

import type { CreateClassificationEvaluatorArgs } from "../types/evals";
import type { ClassificationEvaluator } from "./ClassificationEvaluator";
export interface ToolInvocationEvaluatorArgs<RecordType extends Record<string, unknown> = ToolInvocationEvaluationRecord> extends Omit<CreateClassificationEvaluatorArgs<RecordType>, "promptTemplate" | "choices" | "optimizationDirection" | "name"> {
    optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
    name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
    choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
    promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
}
/**
 * A record to be evaluated by the tool invocation evaluator.
 */
export type ToolInvocationEvaluationRecord = {
    /**
     * The input query or conversation context.
     */
    input: string;
    /**
     * The available tool schemas, either as JSON schema or human-readable format.
     */
    availableTools: string;
    /**
     * The tool invocation(s) made by the LLM, including arguments.
     */
    toolSelection: string;
};
/**
 * Creates a tool invocation evaluator function.
 *
 * This function returns an evaluator that determines whether a tool was invoked
 * correctly with proper arguments, formatting, and safe content.
 *
 * The evaluator checks for:
 * - Properly structured JSON (if applicable)
 * - All required fields/parameters present
 * - No hallucinated or nonexistent fields
 * - Argument values matching user query and schema expectations
 * - No unsafe content (e.g., PII) in arguments
 *
 * @param args - The arguments for creating the tool invocation evaluator.
 * @param args.model - The model to use for classification.
 * @param args.choices - The possible classification choices (defaults to correct/incorrect).
 * @param args.promptTemplate - The prompt template to use (defaults to TOOL_INVOCATION_TEMPLATE).
 * @param args.telemetry - The telemetry to use for the evaluator.
 *
 * @returns An evaluator function that takes a {@link ToolInvocationEvaluationRecord} and returns
 * a classification result indicating whether the tool invocation is correct or incorrect.
 *
 * @example
 * ```ts
 * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
 *
 * // Example with JSON schema format for available tools
 * const result = await evaluator.evaluate({
 *   input: "User: Book a flight from NYC to LA for tomorrow",
 *   availableTools: JSON.stringify({
 *     name: "book_flight",
 *     description: "Book a flight between two cities",
 *     parameters: {
 *       type: "object",
 *       properties: {
 *         origin: { type: "string", description: "Departure city code" },
 *         destination: { type: "string", description: "Arrival city code" },
 *         date: { type: "string", description: "Flight date in YYYY-MM-DD" }
 *       },
 *       required: ["origin", "destination", "date"]
 *     }
 *   }),
 *   toolSelection: 'book_flight(origin="NYC", destination="LA", date="2024-01-15")'
 * });
 * console.log(result.label); // "correct" or "incorrect"
 * ```
 */
export declare function createToolInvocationEvaluator<RecordType extends Record<string, unknown> = ToolInvocationEvaluationRecord>(args: ToolInvocationEvaluatorArgs<RecordType>): ClassificationEvaluator<RecordType>;
//# sourceMappingURL=createToolInvocationEvaluator.d.ts.map