@genkit-ai/vertexai/lib/evaluation/evaluator_factory.mjs

Genkit AI framework plugin for Google Cloud Vertex AI APIs including Gemini APIs, Imagen, and more.

import { GENKIT_CLIENT_HEADER } from "genkit";
import { runInNewSpan } from "genkit/tracing";
class EvaluatorFactory {
  constructor(auth, location, projectId) {
    this.auth = auth;
    this.location = location;
    this.projectId = projectId;
  }
  create(ai, config, toRequest, responseHandler) {
    return ai.defineEvaluator(
      {
        name: `vertexai/${config.metric.toLocaleLowerCase()}`,
        displayName: config.displayName,
        definition: config.definition
      },
      async (datapoint) => {
        const responseSchema = config.responseSchema;
        const response = await this.evaluateInstances(
          ai,
          toRequest(datapoint),
          responseSchema
        );
        return {
          evaluation: responseHandler(response),
          testCaseId: datapoint.testCaseId
        };
      }
    );
  }
  async evaluateInstances(ai, partialRequest, responseSchema) {
    const locationName = `projects/${this.projectId}/locations/${this.location}`;
    return await runInNewSpan(
      ai,
      {
        metadata: {
          name: "EvaluationService#evaluateInstances"
        }
      },
      async (metadata, _otSpan) => {
        const request = {
          location: locationName,
          ...partialRequest
        };
        metadata.input = request;
        const client = await this.auth.getClient();
        const url = `https://${this.location}-aiplatform.googleapis.com/v1beta1/${locationName}:evaluateInstances`;
        const response = await client.request({
          url,
          method: "POST",
          body: JSON.stringify(request),
          headers: {
            "X-Goog-Api-Client": GENKIT_CLIENT_HEADER
          }
        });
        metadata.output = response.data;
        try {
          return responseSchema.parse(response.data);
        } catch (e) {
          throw new Error(`Error parsing ${url} API response: ${e}`);
        }
      }
    );
  }
}
export {
  EvaluatorFactory
};
//# sourceMappingURL=evaluator_factory.mjs.map