@genkit-ai/vertexai/lib/evaluation/evaluation.js

Genkit AI framework plugin for Google Cloud Vertex AI APIs including Gemini APIs, Imagen, and more.

"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
  for (var name in all)
    __defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
  if (from && typeof from === "object" || typeof from === "function") {
    for (let key of __getOwnPropNames(from))
      if (!__hasOwnProp.call(to, key) && key !== except)
        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
  }
  return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var evaluation_exports = {};
__export(evaluation_exports, {
  VertexAIEvaluationMetricType: () => VertexAIEvaluationMetricType,
  vertexEvaluators: () => vertexEvaluators
});
module.exports = __toCommonJS(evaluation_exports);
var import_genkit = require("genkit");
var import_evaluator_factory = require("./evaluator_factory.js");
var VertexAIEvaluationMetricType = /* @__PURE__ */ ((VertexAIEvaluationMetricType2) => {
  VertexAIEvaluationMetricType2["BLEU"] = "BLEU";
  VertexAIEvaluationMetricType2["ROUGE"] = "ROUGE";
  VertexAIEvaluationMetricType2["FLUENCY"] = "FLEUNCY";
  VertexAIEvaluationMetricType2["SAFETY"] = "SAFETY";
  VertexAIEvaluationMetricType2["GROUNDEDNESS"] = "GROUNDEDNESS";
  VertexAIEvaluationMetricType2["SUMMARIZATION_QUALITY"] = "SUMMARIZATION_QUALITY";
  VertexAIEvaluationMetricType2["SUMMARIZATION_HELPFULNESS"] = "SUMMARIZATION_HELPFULNESS";
  VertexAIEvaluationMetricType2["SUMMARIZATION_VERBOSITY"] = "SUMMARIZATION_VERBOSITY";
  return VertexAIEvaluationMetricType2;
})(VertexAIEvaluationMetricType || {});
function stringify(input) {
  return typeof input === "string" ? input : JSON.stringify(input);
}
function vertexEvaluators(ai, auth, metrics, projectId, location) {
  const factory = new import_evaluator_factory.EvaluatorFactory(auth, location, projectId);
  return metrics.map((metric) => {
    const metricType = isConfig(metric) ? metric.type : metric;
    const metricSpec = isConfig(metric) ? metric.metricSpec : {};
    switch (metricType) {
      case "BLEU" /* BLEU */: {
        return createBleuEvaluator(ai, factory, metricSpec);
      }
      case "ROUGE" /* ROUGE */: {
        return createRougeEvaluator(ai, factory, metricSpec);
      }
      case "FLEUNCY" /* FLUENCY */: {
        return createFluencyEvaluator(ai, factory, metricSpec);
      }
      case "SAFETY" /* SAFETY */: {
        return createSafetyEvaluator(ai, factory, metricSpec);
      }
      case "GROUNDEDNESS" /* GROUNDEDNESS */: {
        return createGroundednessEvaluator(ai, factory, metricSpec);
      }
      case "SUMMARIZATION_QUALITY" /* SUMMARIZATION_QUALITY */: {
        return createSummarizationQualityEvaluator(ai, factory, metricSpec);
      }
      case "SUMMARIZATION_HELPFULNESS" /* SUMMARIZATION_HELPFULNESS */: {
        return createSummarizationHelpfulnessEvaluator(ai, factory, metricSpec);
      }
      case "SUMMARIZATION_VERBOSITY" /* SUMMARIZATION_VERBOSITY */: {
        return createSummarizationVerbosityEvaluator(ai, factory, metricSpec);
      }
    }
  });
}
function isConfig(config) {
  return config.type !== void 0;
}
const BleuResponseSchema = import_genkit.z.object({
  bleuResults: import_genkit.z.object({
    bleuMetricValues: import_genkit.z.array(import_genkit.z.object({ score: import_genkit.z.number() }))
  })
});
function createBleuEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "BLEU" /* BLEU */,
      displayName: "BLEU",
      definition: "Computes the BLEU score by comparing the output against the ground truth",
      responseSchema: BleuResponseSchema
    },
    (datapoint) => {
      return {
        bleuInput: {
          metricSpec,
          instances: [
            {
              prediction: stringify(datapoint.output),
              reference: datapoint.reference
            }
          ]
        }
      };
    },
    (response) => {
      return {
        score: response.bleuResults.bleuMetricValues[0].score
      };
    }
  );
}
const RougeResponseSchema = import_genkit.z.object({
  rougeResults: import_genkit.z.object({
    rougeMetricValues: import_genkit.z.array(import_genkit.z.object({ score: import_genkit.z.number() }))
  })
});
function createRougeEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "ROUGE" /* ROUGE */,
      displayName: "ROUGE",
      definition: "Computes the ROUGE score by comparing the output against the ground truth",
      responseSchema: RougeResponseSchema
    },
    (datapoint) => {
      return {
        rougeInput: {
          metricSpec,
          instances: {
            prediction: stringify(datapoint.output),
            reference: datapoint.reference
          }
        }
      };
    },
    (response) => {
      return {
        score: response.rougeResults.rougeMetricValues[0].score
      };
    }
  );
}
const FluencyResponseSchema = import_genkit.z.object({
  fluencyResult: import_genkit.z.object({
    score: import_genkit.z.number(),
    explanation: import_genkit.z.string(),
    confidence: import_genkit.z.number()
  })
});
function createFluencyEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "FLEUNCY" /* FLUENCY */,
      displayName: "Fluency",
      definition: "Assesses the language mastery of an output",
      responseSchema: FluencyResponseSchema
    },
    (datapoint) => {
      return {
        fluencyInput: {
          metricSpec,
          instance: {
            prediction: stringify(datapoint.output)
          }
        }
      };
    },
    (response) => {
      return {
        score: response.fluencyResult.score,
        details: {
          reasoning: response.fluencyResult.explanation
        }
      };
    }
  );
}
const SafetyResponseSchema = import_genkit.z.object({
  safetyResult: import_genkit.z.object({
    score: import_genkit.z.number(),
    explanation: import_genkit.z.string(),
    confidence: import_genkit.z.number()
  })
});
function createSafetyEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "SAFETY" /* SAFETY */,
      displayName: "Safety",
      definition: "Assesses the level of safety of an output",
      responseSchema: SafetyResponseSchema
    },
    (datapoint) => {
      return {
        safetyInput: {
          metricSpec,
          instance: {
            prediction: stringify(datapoint.output)
          }
        }
      };
    },
    (response) => {
      return {
        score: response.safetyResult.score,
        details: {
          reasoning: response.safetyResult.explanation
        }
      };
    }
  );
}
const GroundednessResponseSchema = import_genkit.z.object({
  groundednessResult: import_genkit.z.object({
    score: import_genkit.z.number(),
    explanation: import_genkit.z.string(),
    confidence: import_genkit.z.number()
  })
});
function createGroundednessEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "GROUNDEDNESS" /* GROUNDEDNESS */,
      displayName: "Groundedness",
      definition: "Assesses the ability to provide or reference information included only in the context",
      responseSchema: GroundednessResponseSchema
    },
    (datapoint) => {
      return {
        groundednessInput: {
          metricSpec,
          instance: {
            prediction: stringify(datapoint.output),
            context: datapoint.context?.join(". ")
          }
        }
      };
    },
    (response) => {
      return {
        score: response.groundednessResult.score,
        details: {
          reasoning: response.groundednessResult.explanation
        }
      };
    }
  );
}
const SummarizationQualityResponseSchema = import_genkit.z.object({
  summarizationQualityResult: import_genkit.z.object({
    score: import_genkit.z.number(),
    explanation: import_genkit.z.string(),
    confidence: import_genkit.z.number()
  })
});
function createSummarizationQualityEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "SUMMARIZATION_QUALITY" /* SUMMARIZATION_QUALITY */,
      displayName: "Summarization quality",
      definition: "Assesses the overall ability to summarize text",
      responseSchema: SummarizationQualityResponseSchema
    },
    (datapoint) => {
      return {
        summarizationQualityInput: {
          metricSpec,
          instance: {
            prediction: stringify(datapoint.output),
            instruction: stringify(datapoint.input),
            context: datapoint.context?.join(". ")
          }
        }
      };
    },
    (response) => {
      return {
        score: response.summarizationQualityResult.score,
        details: {
          reasoning: response.summarizationQualityResult.explanation
        }
      };
    }
  );
}
const SummarizationHelpfulnessResponseSchema = import_genkit.z.object({
  summarizationHelpfulnessResult: import_genkit.z.object({
    score: import_genkit.z.number(),
    explanation: import_genkit.z.string(),
    confidence: import_genkit.z.number()
  })
});
function createSummarizationHelpfulnessEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "SUMMARIZATION_HELPFULNESS" /* SUMMARIZATION_HELPFULNESS */,
      displayName: "Summarization helpfulness",
      definition: "Assesses the ability to provide a summarization, which contains the details necessary to substitute the original text",
      responseSchema: SummarizationHelpfulnessResponseSchema
    },
    (datapoint) => {
      return {
        summarizationHelpfulnessInput: {
          metricSpec,
          instance: {
            prediction: stringify(datapoint.output),
            instruction: stringify(datapoint.input),
            context: datapoint.context?.join(". ")
          }
        }
      };
    },
    (response) => {
      return {
        score: response.summarizationHelpfulnessResult.score,
        details: {
          reasoning: response.summarizationHelpfulnessResult.explanation
        }
      };
    }
  );
}
const SummarizationVerbositySchema = import_genkit.z.object({
  summarizationVerbosityResult: import_genkit.z.object({
    score: import_genkit.z.number(),
    explanation: import_genkit.z.string(),
    confidence: import_genkit.z.number()
  })
});
function createSummarizationVerbosityEvaluator(ai, factory, metricSpec) {
  return factory.create(
    ai,
    {
      metric: "SUMMARIZATION_VERBOSITY" /* SUMMARIZATION_VERBOSITY */,
      displayName: "Summarization verbosity",
      definition: "Aassess the ability to provide a succinct summarization",
      responseSchema: SummarizationVerbositySchema
    },
    (datapoint) => {
      return {
        summarizationVerbosityInput: {
          metricSpec,
          instance: {
            prediction: stringify(datapoint.output),
            instruction: stringify(datapoint.input),
            context: datapoint.context?.join(". ")
          }
        }
      };
    },
    (response) => {
      return {
        score: response.summarizationVerbosityResult.score,
        details: {
          reasoning: response.summarizationVerbosityResult.explanation
        }
      };
    }
  );
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
  VertexAIEvaluationMetricType,
  vertexEvaluators
});
//# sourceMappingURL=evaluation.js.map