@arizeai/phoenix-evals/src/__generated__/default_templates/TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG.ts

A library for running evaluations for AI use cases

// This file is generated. Do not edit by hand.

import type { ClassificationEvaluatorConfig } from "../types";

export const TOOL_SELECTION_CLASSIFICATION_EVALUATOR_CONFIG: ClassificationEvaluatorConfig = {
  name: "tool_selection",
  description: "For determining if the correct tool was selected for a given context.",
  optimizationDirection: "MAXIMIZE",
  template: [
    {
      role: "user",
      content: `
You are an impartial judge evaluating an LLM's tool-calling behavior, specifically whether the LLM selected the most appropriate tool or tools for the task.

Your task: Determine whether the LLM's tool selection was correct or incorrect based on:
- The conversation context (input)
- The available tools
- The LLM's output and tool invocation(s)

Criteria
Return "correct" only when ALL of the following are true:
- The LLM chose the best available tool for the user query OR correctly avoided tools if none were needed.
- The tool name exists in the available tools list.
- The tool is allowed and safe to call.
- The LLM selected the correct number of tools for the task.

Return "incorrect" if ANY of the following are true:
- The LLM used a hallucinated or nonexistent tool.
- The LLM selected a tool when none was needed.
- The LLM did not use a tool when one was required.
- The LLM chose a suboptimal or irrelevant tool.
- The LLM selected an unsafe or not-permitted tool.
- The tool name does not appear in the available tools list.

Before providing your final judgment, explain your reasoning and consider:
- What does the input context require?
- Can this be answered without tools, or is a tool necessary?
- If a tool was selected, does it exist in the available tools?
- Does the selected tool's description match the user's needs?
- Is the selection safe and appropriate?
- Is there a better tool available that should have been chosen instead?

<data>
<input>
{{input}}
</input>

<available_tools>
{{availableTools}}
</available_tools>

<output>
{{toolSelection}}
</output>
</data>

Given the above data, is the tool selection correct or incorrect?
`,
    },
  ],
  choices: {
  "correct": 1,
  "incorrect": 0
},
};