@arizeai/phoenix-evals
Version:
A library for running evaluations for AI use cases
101 lines • 3.81 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.createEvaluator = createEvaluator;
const openinference_core_1 = require("@arizeai/openinference-core");
const FunctionEvaluator_1 = require("../core/FunctionEvaluator");
const telemetry_1 = require("../telemetry");
const asEvaluatorFn_1 = require("./asEvaluatorFn");
function generateUniqueName() {
return `evaluator-${Math.random().toString(36).substring(2, 15)}`;
}
/**
* A factory function for creating a custom evaluator from any function.
*
* This function wraps a user-provided function into an evaluator that can be used
* with Phoenix experiments and evaluations. The function can be synchronous or
* asynchronous, and can return a number, an {@link EvaluationResult} object, or
* a value that will be automatically converted to an evaluation result.
*
* The evaluator will automatically:
* - Convert the function's return value to an {@link EvaluationResult}
* - Handle both sync and async functions
* - Wrap the function with OpenTelemetry spans if telemetry is enabled
* - Infer the evaluator name from the function name if not provided
*
* @typeParam RecordType - The type of the input record that the evaluator expects.
* Must extend `Record<string, unknown>`.
* @typeParam Fn - The type of the function being wrapped. Must be a function that
* accepts the record type and returns a value compatible with {@link EvaluationResult}.
*
* @param fn - The function to wrap as an evaluator. Can be synchronous or asynchronous.
* The function should accept a record of type `RecordType` and return either:
* - A number (will be converted to `{ score: number }`)
* - An {@link EvaluationResult} object
* - Any value that can be converted to an evaluation result
*
* @param options - Optional configuration for the evaluator. See {@link CreateEvaluatorOptions}
* for details on available options.
*
* @returns An {@link EvaluatorInterface} that can be used with Phoenix experiments
* and evaluation workflows.
*
* @example
* Basic usage with a simple scoring function:
* ```typescript
* const accuracyEvaluator = createEvaluator(
* ({ output, expected }) => {
* return output === expected ? 1 : 0;
* },
* {
* name: "accuracy",
* kind: "CODE",
* optimizationDirection: "MAXIMIZE"
* }
* );
*
* const result = await accuracyEvaluator.evaluate({
* output: "correct answer",
* expected: "correct answer"
* });
* // result: { score: 1 }
* ```
*
*
* @example
* Returning a full EvaluationResult:
* ```typescript
* const qualityEvaluator = createEvaluator(
* ({ output }) => {
* const score = calculateQuality(output);
* return {
* score,
* label: score > 0.8 ? "high" : "low",
* explanation: `Quality score: ${score}`
* };
* },
* { name: "quality" }
* );
* ```
*/
function createEvaluator(fn, options) {
var _a;
const { name, kind, optimizationDirection, telemetry = { isEnabled: true }, } = options || {};
const evaluatorName = name || fn.name || generateUniqueName();
let evaluateFn = (0, asEvaluatorFn_1.asEvaluatorFn)(fn);
// Add OpenTelemetry span wrapping if telemetry is enabled
if (telemetry && telemetry.isEnabled) {
evaluateFn = (0, openinference_core_1.withSpan)(evaluateFn, {
tracer: (_a = telemetry.tracer) !== null && _a !== void 0 ? _a : telemetry_1.tracer,
name: evaluatorName,
kind: "EVALUATOR",
});
}
return new FunctionEvaluator_1.FunctionEvaluator({
evaluateFn,
name: evaluatorName,
kind: kind || "CODE",
optimizationDirection: optimizationDirection || "MAXIMIZE",
telemetry,
});
}
//# sourceMappingURL=createEvaluator.js.map