@genkit-ai/vertexai/lib/evaluation/evaluation.mjs.map

Genkit AI framework plugin for Google Cloud Vertex AI APIs including Gemini APIs, Imagen, and more.

{"version":3,"sources":["../../src/evaluation/evaluation.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Action, Genkit, z } from 'genkit';\nimport { GoogleAuth } from 'google-auth-library';\nimport { EvaluatorFactory } from './evaluator_factory.js';\n\n/**\n * Vertex AI Evaluation metrics. See API documentation for more information.\n * https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation#parameter-list\n */\nexport enum VertexAIEvaluationMetricType {\n  // Update genkit/docs/plugins/vertex-ai.md when modifying the list of enums\n  BLEU = 'BLEU',\n  ROUGE = 'ROUGE',\n  FLUENCY = 'FLEUNCY',\n  SAFETY = 'SAFETY',\n  GROUNDEDNESS = 'GROUNDEDNESS',\n  SUMMARIZATION_QUALITY = 'SUMMARIZATION_QUALITY',\n  SUMMARIZATION_HELPFULNESS = 'SUMMARIZATION_HELPFULNESS',\n  SUMMARIZATION_VERBOSITY = 'SUMMARIZATION_VERBOSITY',\n}\n\n/**\n * Evaluation metric config. Use `metricSpec` to define the behavior of the metric.\n * The value of `metricSpec` will be included in the request to the API. See the API documentation\n * for details on the possible values of `metricSpec` for each metric.\n * https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation#parameter-list\n */\nexport type VertexAIEvaluationMetricConfig = {\n  type: VertexAIEvaluationMetricType;\n  metricSpec: any;\n};\n\nexport type VertexAIEvaluationMetric =\n  | VertexAIEvaluationMetricType\n  | VertexAIEvaluationMetricConfig;\n\nfunction stringify(input: unknown) {\n  return typeof input === 'string' ? input : JSON.stringify(input);\n}\n\nexport function vertexEvaluators(\n  ai: Genkit,\n  auth: GoogleAuth,\n  metrics: VertexAIEvaluationMetric[],\n  projectId: string,\n  location: string\n): Action[] {\n  const factory = new EvaluatorFactory(auth, location, projectId);\n  return metrics.map((metric) => {\n    const metricType = isConfig(metric) ? metric.type : metric;\n    const metricSpec = isConfig(metric) ? metric.metricSpec : {};\n\n    switch (metricType) {\n      case VertexAIEvaluationMetricType.BLEU: {\n        return createBleuEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.ROUGE: {\n        return createRougeEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.FLUENCY: {\n        return createFluencyEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.SAFETY: {\n        return createSafetyEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.GROUNDEDNESS: {\n        return createGroundednessEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.SUMMARIZATION_QUALITY: {\n        return createSummarizationQualityEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.SUMMARIZATION_HELPFULNESS: {\n        return createSummarizationHelpfulnessEvaluator(ai, factory, metricSpec);\n      }\n      case VertexAIEvaluationMetricType.SUMMARIZATION_VERBOSITY: {\n        return createSummarizationVerbosityEvaluator(ai, factory, metricSpec);\n      }\n    }\n  });\n}\n\nfunction isConfig(\n  config: VertexAIEvaluationMetric\n): config is VertexAIEvaluationMetricConfig {\n  return (config as VertexAIEvaluationMetricConfig).type !== undefined;\n}\n\nconst BleuResponseSchema = z.object({\n  bleuResults: z.object({\n    bleuMetricValues: z.array(z.object({ score: z.number() })),\n  }),\n});\n\n// TODO: Add support for batch inputs\nfunction createBleuEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.BLEU,\n      displayName: 'BLEU',\n      definition:\n        'Computes the BLEU score by comparing the output against the ground truth',\n      responseSchema: BleuResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        bleuInput: {\n          metricSpec,\n          instances: [\n            {\n              prediction: stringify(datapoint.output),\n              reference: datapoint.reference,\n            },\n          ],\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.bleuResults.bleuMetricValues[0].score,\n      };\n    }\n  );\n}\n\nconst RougeResponseSchema = z.object({\n  rougeResults: z.object({\n    rougeMetricValues: z.array(z.object({ score: z.number() })),\n  }),\n});\n\n// TODO: Add support for batch inputs\nfunction createRougeEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.ROUGE,\n      displayName: 'ROUGE',\n      definition:\n        'Computes the ROUGE score by comparing the output against the ground truth',\n      responseSchema: RougeResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        rougeInput: {\n          metricSpec,\n          instances: {\n            prediction: stringify(datapoint.output),\n            reference: datapoint.reference,\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.rougeResults.rougeMetricValues[0].score,\n      };\n    }\n  );\n}\n\nconst FluencyResponseSchema = z.object({\n  fluencyResult: z.object({\n    score: z.number(),\n    explanation: z.string(),\n    confidence: z.number(),\n  }),\n});\n\nfunction createFluencyEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.FLUENCY,\n      displayName: 'Fluency',\n      definition: 'Assesses the language mastery of an output',\n      responseSchema: FluencyResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        fluencyInput: {\n          metricSpec,\n          instance: {\n            prediction: stringify(datapoint.output),\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.fluencyResult.score,\n        details: {\n          reasoning: response.fluencyResult.explanation,\n        },\n      };\n    }\n  );\n}\n\nconst SafetyResponseSchema = z.object({\n  safetyResult: z.object({\n    score: z.number(),\n    explanation: z.string(),\n    confidence: z.number(),\n  }),\n});\n\nfunction createSafetyEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.SAFETY,\n      displayName: 'Safety',\n      definition: 'Assesses the level of safety of an output',\n      responseSchema: SafetyResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        safetyInput: {\n          metricSpec,\n          instance: {\n            prediction: stringify(datapoint.output),\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.safetyResult.score,\n        details: {\n          reasoning: response.safetyResult.explanation,\n        },\n      };\n    }\n  );\n}\n\nconst GroundednessResponseSchema = z.object({\n  groundednessResult: z.object({\n    score: z.number(),\n    explanation: z.string(),\n    confidence: z.number(),\n  }),\n});\n\nfunction createGroundednessEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.GROUNDEDNESS,\n      displayName: 'Groundedness',\n      definition:\n        'Assesses the ability to provide or reference information included only in the context',\n      responseSchema: GroundednessResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        groundednessInput: {\n          metricSpec,\n          instance: {\n            prediction: stringify(datapoint.output),\n            context: datapoint.context?.join('. '),\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.groundednessResult.score,\n        details: {\n          reasoning: response.groundednessResult.explanation,\n        },\n      };\n    }\n  );\n}\n\nconst SummarizationQualityResponseSchema = z.object({\n  summarizationQualityResult: z.object({\n    score: z.number(),\n    explanation: z.string(),\n    confidence: z.number(),\n  }),\n});\n\nfunction createSummarizationQualityEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.SUMMARIZATION_QUALITY,\n      displayName: 'Summarization quality',\n      definition: 'Assesses the overall ability to summarize text',\n      responseSchema: SummarizationQualityResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        summarizationQualityInput: {\n          metricSpec,\n          instance: {\n            prediction: stringify(datapoint.output),\n            instruction: stringify(datapoint.input),\n            context: datapoint.context?.join('. '),\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.summarizationQualityResult.score,\n        details: {\n          reasoning: response.summarizationQualityResult.explanation,\n        },\n      };\n    }\n  );\n}\n\nconst SummarizationHelpfulnessResponseSchema = z.object({\n  summarizationHelpfulnessResult: z.object({\n    score: z.number(),\n    explanation: z.string(),\n    confidence: z.number(),\n  }),\n});\n\nfunction createSummarizationHelpfulnessEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.SUMMARIZATION_HELPFULNESS,\n      displayName: 'Summarization helpfulness',\n      definition:\n        'Assesses the ability to provide a summarization, which contains the details necessary to substitute the original text',\n      responseSchema: SummarizationHelpfulnessResponseSchema,\n    },\n    (datapoint) => {\n      return {\n        summarizationHelpfulnessInput: {\n          metricSpec,\n          instance: {\n            prediction: stringify(datapoint.output),\n            instruction: stringify(datapoint.input),\n            context: datapoint.context?.join('. '),\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.summarizationHelpfulnessResult.score,\n        details: {\n          reasoning: response.summarizationHelpfulnessResult.explanation,\n        },\n      };\n    }\n  );\n}\n\nconst SummarizationVerbositySchema = z.object({\n  summarizationVerbosityResult: z.object({\n    score: z.number(),\n    explanation: z.string(),\n    confidence: z.number(),\n  }),\n});\n\nfunction createSummarizationVerbosityEvaluator(\n  ai: Genkit,\n  factory: EvaluatorFactory,\n  metricSpec: any\n): Action {\n  return factory.create(\n    ai,\n    {\n      metric: VertexAIEvaluationMetricType.SUMMARIZATION_VERBOSITY,\n      displayName: 'Summarization verbosity',\n      definition: 'Aassess the ability to provide a succinct summarization',\n      responseSchema: SummarizationVerbositySchema,\n    },\n    (datapoint) => {\n      return {\n        summarizationVerbosityInput: {\n          metricSpec,\n          instance: {\n            prediction: stringify(datapoint.output),\n            instruction: stringify(datapoint.input),\n            context: datapoint.context?.join('. '),\n          },\n        },\n      };\n    },\n    (response) => {\n      return {\n        score: response.summarizationVerbosityResult.score,\n        details: {\n          reasoning: response.summarizationVerbosityResult.explanation,\n        },\n      };\n    }\n  );\n}\n"],"mappings":"AAgBA,SAAyB,SAAS;AAElC,SAAS,wBAAwB;AAM1B,IAAK,+BAAL,kBAAKA,kCAAL;AAEL,EAAAA,8BAAA,UAAO;AACP,EAAAA,8BAAA,WAAQ;AACR,EAAAA,8BAAA,aAAU;AACV,EAAAA,8BAAA,YAAS;AACT,EAAAA,8BAAA,kBAAe;AACf,EAAAA,8BAAA,2BAAwB;AACxB,EAAAA,8BAAA,+BAA4B;AAC5B,EAAAA,8BAAA,6BAA0B;AAThB,SAAAA;AAAA,GAAA;AA2BZ,SAAS,UAAU,OAAgB;AACjC,SAAO,OAAO,UAAU,WAAW,QAAQ,KAAK,UAAU,KAAK;AACjE;AAEO,SAAS,iBACd,IACA,MACA,SACA,WACA,UACU;AACV,QAAM,UAAU,IAAI,iBAAiB,MAAM,UAAU,SAAS;AAC9D,SAAO,QAAQ,IAAI,CAAC,WAAW;AAC7B,UAAM,aAAa,SAAS,MAAM,IAAI,OAAO,OAAO;AACpD,UAAM,aAAa,SAAS,MAAM,IAAI,OAAO,aAAa,CAAC;AAE3D,YAAQ,YAAY;AAAA,MAClB,KAAK,mBAAmC;AACtC,eAAO,oBAAoB,IAAI,SAAS,UAAU;AAAA,MACpD;AAAA,MACA,KAAK,qBAAoC;AACvC,eAAO,qBAAqB,IAAI,SAAS,UAAU;AAAA,MACrD;AAAA,MACA,KAAK,yBAAsC;AACzC,eAAO,uBAAuB,IAAI,SAAS,UAAU;AAAA,MACvD;AAAA,MACA,KAAK,uBAAqC;AACxC,eAAO,sBAAsB,IAAI,SAAS,UAAU;AAAA,MACtD;AAAA,MACA,KAAK,mCAA2C;AAC9C,eAAO,4BAA4B,IAAI,SAAS,UAAU;AAAA,MAC5D;AAAA,MACA,KAAK,qDAAoD;AACvD,eAAO,oCAAoC,IAAI,SAAS,UAAU;AAAA,MACpE;AAAA,MACA,KAAK,6DAAwD;AAC3D,eAAO,wCAAwC,IAAI,SAAS,UAAU;AAAA,MACxE;AAAA,MACA,KAAK,yDAAsD;AACzD,eAAO,sCAAsC,IAAI,SAAS,UAAU;AAAA,MACtE;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,SACP,QAC0C;AAC1C,SAAQ,OAA0C,SAAS;AAC7D;AAEA,MAAM,qBAAqB,EAAE,OAAO;AAAA,EAClC,aAAa,EAAE,OAAO;AAAA,IACpB,kBAAkB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AAAA,EAC3D,CAAC;AACH,CAAC;AAGD,SAAS,oBACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YACE;AAAA,MACF,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,WAAW;AAAA,UACT;AAAA,UACA,WAAW;AAAA,YACT;AAAA,cACE,YAAY,UAAU,UAAU,MAAM;AAAA,cACtC,WAAW,UAAU;AAAA,YACvB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,YAAY,iBAAiB,CAAC,EAAE;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,sBAAsB,EAAE,OAAO;AAAA,EACnC,cAAc,EAAE,OAAO;AAAA,IACrB,mBAAmB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AAAA,EAC5D,CAAC;AACH,CAAC;AAGD,SAAS,qBACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YACE;AAAA,MACF,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,YAAY;AAAA,UACV;AAAA,UACA,WAAW;AAAA,YACT,YAAY,UAAU,UAAU,MAAM;AAAA,YACtC,WAAW,UAAU;AAAA,UACvB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,aAAa,kBAAkB,CAAC,EAAE;AAAA,MACpD;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,wBAAwB,EAAE,OAAO;AAAA,EACrC,eAAe,EAAE,OAAO;AAAA,IACtB,OAAO,EAAE,OAAO;AAAA,IAChB,aAAa,EAAE,OAAO;AAAA,IACtB,YAAY,EAAE,OAAO;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,uBACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,cAAc;AAAA,UACZ;AAAA,UACA,UAAU;AAAA,YACR,YAAY,UAAU,UAAU,MAAM;AAAA,UACxC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,cAAc;AAAA,QAC9B,SAAS;AAAA,UACP,WAAW,SAAS,cAAc;AAAA,QACpC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,uBAAuB,EAAE,OAAO;AAAA,EACpC,cAAc,EAAE,OAAO;AAAA,IACrB,OAAO,EAAE,OAAO;AAAA,IAChB,aAAa,EAAE,OAAO;AAAA,IACtB,YAAY,EAAE,OAAO;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,sBACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,aAAa;AAAA,UACX;AAAA,UACA,UAAU;AAAA,YACR,YAAY,UAAU,UAAU,MAAM;AAAA,UACxC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,aAAa;AAAA,QAC7B,SAAS;AAAA,UACP,WAAW,SAAS,aAAa;AAAA,QACnC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,6BAA6B,EAAE,OAAO;AAAA,EAC1C,oBAAoB,EAAE,OAAO;AAAA,IAC3B,OAAO,EAAE,OAAO;AAAA,IAChB,aAAa,EAAE,OAAO;AAAA,IACtB,YAAY,EAAE,OAAO;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,4BACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YACE;AAAA,MACF,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,mBAAmB;AAAA,UACjB;AAAA,UACA,UAAU;AAAA,YACR,YAAY,UAAU,UAAU,MAAM;AAAA,YACtC,SAAS,UAAU,SAAS,KAAK,IAAI;AAAA,UACvC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,mBAAmB;AAAA,QACnC,SAAS;AAAA,UACP,WAAW,SAAS,mBAAmB;AAAA,QACzC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,qCAAqC,EAAE,OAAO;AAAA,EAClD,4BAA4B,EAAE,OAAO;AAAA,IACnC,OAAO,EAAE,OAAO;AAAA,IAChB,aAAa,EAAE,OAAO;AAAA,IACtB,YAAY,EAAE,OAAO;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,oCACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,2BAA2B;AAAA,UACzB;AAAA,UACA,UAAU;AAAA,YACR,YAAY,UAAU,UAAU,MAAM;AAAA,YACtC,aAAa,UAAU,UAAU,KAAK;AAAA,YACtC,SAAS,UAAU,SAAS,KAAK,IAAI;AAAA,UACvC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,2BAA2B;AAAA,QAC3C,SAAS;AAAA,UACP,WAAW,SAAS,2BAA2B;AAAA,QACjD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,yCAAyC,EAAE,OAAO;AAAA,EACtD,gCAAgC,EAAE,OAAO;AAAA,IACvC,OAAO,EAAE,OAAO;AAAA,IAChB,aAAa,EAAE,OAAO;AAAA,IACtB,YAAY,EAAE,OAAO;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,wCACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YACE;AAAA,MACF,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,+BAA+B;AAAA,UAC7B;AAAA,UACA,UAAU;AAAA,YACR,YAAY,UAAU,UAAU,MAAM;AAAA,YACtC,aAAa,UAAU,UAAU,KAAK;AAAA,YACtC,SAAS,UAAU,SAAS,KAAK,IAAI;AAAA,UACvC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,+BAA+B;AAAA,QAC/C,SAAS;AAAA,UACP,WAAW,SAAS,+BAA+B;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,MAAM,+BAA+B,EAAE,OAAO;AAAA,EAC5C,8BAA8B,EAAE,OAAO;AAAA,IACrC,OAAO,EAAE,OAAO;AAAA,IAChB,aAAa,EAAE,OAAO;AAAA,IACtB,YAAY,EAAE,OAAO;AAAA,EACvB,CAAC;AACH,CAAC;AAED,SAAS,sCACP,IACA,SACA,YACQ;AACR,SAAO,QAAQ;AAAA,IACb;AAAA,IACA;AAAA,MACE,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,IACA,CAAC,cAAc;AACb,aAAO;AAAA,QACL,6BAA6B;AAAA,UAC3B;AAAA,UACA,UAAU;AAAA,YACR,YAAY,UAAU,UAAU,MAAM;AAAA,YACtC,aAAa,UAAU,UAAU,KAAK;AAAA,YACtC,SAAS,UAAU,SAAS,KAAK,IAAI;AAAA,UACvC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,IACA,CAAC,aAAa;AACZ,aAAO;AAAA,QACL,OAAO,SAAS,6BAA6B;AAAA,QAC7C,SAAS;AAAA,UACP,WAAW,SAAS,6BAA6B;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;","names":["VertexAIEvaluationMetricType"]}