@arizeai/phoenix-evals/src/llm/createFaithfulnessEvaluator.ts

A library for running evaluations for AI use cases

import { FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG } from "../__generated__/default_templates";
import type { CreateClassificationEvaluatorArgs } from "../types/evals";
import type { ClassificationEvaluator } from "./ClassificationEvaluator";
import { createClassificationEvaluator } from "./createClassificationEvaluator";

export interface FaithfulnessEvaluatorArgs<
  RecordType extends Record<string, unknown> = FaithfulnessEvaluationRecord,
> extends Omit<
  CreateClassificationEvaluatorArgs<RecordType>,
  "promptTemplate" | "choices" | "optimizationDirection" | "name"
> {
  optimizationDirection?: CreateClassificationEvaluatorArgs<RecordType>["optimizationDirection"];
  name?: CreateClassificationEvaluatorArgs<RecordType>["name"];
  choices?: CreateClassificationEvaluatorArgs<RecordType>["choices"];
  promptTemplate?: CreateClassificationEvaluatorArgs<RecordType>["promptTemplate"];
}

/**
 * A record to be evaluated by the faithfulness evaluator.
 */
export type FaithfulnessEvaluationRecord = {
  input: string;
  output: string;
  context?: string;
};
/**
 * Creates a function that evaluates whether an answer is faithful or unfaithful based on a query and reference text.
 *
 * @param args - The arguments for creating the faithfulness evaluator.
 * @returns A function that evaluates whether an answer is faithful or unfaithful based on a query and reference text.
 */
export function createFaithfulnessEvaluator<
  RecordType extends Record<string, unknown> = FaithfulnessEvaluationRecord,
>(
  args: FaithfulnessEvaluatorArgs<RecordType>
): ClassificationEvaluator<RecordType> {
  const {
    choices = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.choices,
    promptTemplate = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.template,
    optimizationDirection = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.optimizationDirection,
    name = FAITHFULNESS_CLASSIFICATION_EVALUATOR_CONFIG.name,
    ...rest
  } = args;
  return createClassificationEvaluator<RecordType>({
    ...rest,
    promptTemplate,
    choices,
    optimizationDirection,
    name,
  });
}