@genkit-ai/ai/lib/evaluator.mjs.map

Genkit AI framework generative AI APIs.

{"version":3,"sources":["../src/evaluator.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { action, z, type Action } from '@genkit-ai/core';\nimport { logger } from '@genkit-ai/core/logging';\nimport type { Registry } from '@genkit-ai/core/registry';\nimport { toJsonSchema } from '@genkit-ai/core/schema';\nimport { SpanMetadata, runInNewSpan } from '@genkit-ai/core/tracing';\nimport { randomUUID } from 'crypto';\n\nexport const ATTR_PREFIX = 'genkit';\nexport const SPAN_STATE_ATTR = ATTR_PREFIX + ':state';\n\n/** Zod schema for a base evaluation data point containing input, output, context, and reference fields. */\nexport const BaseDataPointSchema = z.object({\n  input: z.unknown(),\n  output: z.unknown().optional(),\n  context: z.array(z.unknown()).optional(),\n  reference: z.unknown().optional(),\n  testCaseId: z.string().optional(),\n  traceIds: z.array(z.string()).optional(),\n});\n\n/** Zod schema for an evaluation data point used in evaluator actions. Requires `testCaseId` to be present. */\nexport const BaseEvalDataPointSchema = BaseDataPointSchema.extend({\n  testCaseId: z.string(),\n});\nexport type BaseEvalDataPoint = z.infer<typeof BaseEvalDataPointSchema>;\n\nconst EvalStatusEnumSchema = z.enum(['UNKNOWN', 'PASS', 'FAIL']);\n\n/** Enum that indicates if an evaluation has passed or failed */\nexport enum EvalStatusEnum {\n  UNKNOWN = 'UNKNOWN',\n  PASS = 'PASS',\n  FAIL = 'FAIL',\n}\n\n/** Zod schema for an evaluation score, including optional numeric/string/boolean score, status, and details. */\nexport const ScoreSchema = z.object({\n  id: z\n    .string()\n    .describe(\n      'Optional ID to differentiate different scores if applying in a single evaluation'\n    )\n    .optional(),\n  score: z.union([z.number(), z.string(), z.boolean()]).optional(),\n  status: EvalStatusEnumSchema.optional(),\n  error: z.string().optional(),\n  details: z\n    .object({\n      reasoning: z.string().optional(),\n    })\n    .passthrough()\n    .optional(),\n});\n\n// Update genkit-tools/src/utils/evals.ts if you change this value\nexport const EVALUATOR_METADATA_KEY_DISPLAY_NAME = 'evaluatorDisplayName';\nexport const EVALUATOR_METADATA_KEY_DEFINITION = 'evaluatorDefinition';\nexport const EVALUATOR_METADATA_KEY_IS_BILLED = 'evaluatorIsBilled';\n\nexport type Score = z.infer<typeof ScoreSchema>;\nexport type BaseDataPoint = z.infer<typeof BaseDataPointSchema>;\nexport type Dataset<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n> = Array<z.infer<DataPoint>>;\n\n/** Zod schema for a single evaluation response, containing test case ID and evaluation score(s). */\nexport const EvalResponseSchema = z.object({\n  sampleIndex: z.number().optional(),\n  testCaseId: z.string(),\n  traceId: z.string().optional(),\n  spanId: z.string().optional(),\n  evaluation: z.union([ScoreSchema, z.array(ScoreSchema)]),\n});\nexport type EvalResponse = z.infer<typeof EvalResponseSchema>;\n\n/** Zod schema for an array of {@link EvalResponse} objects. */\nexport const EvalResponsesSchema = z.array(EvalResponseSchema);\nexport type EvalResponses = z.infer<typeof EvalResponsesSchema>;\n\n/** Implementation function for an evaluator. Receives a data point and optional config, returns an {@link EvalResponse}. */\nexport type EvaluatorFn<\n  EvalDataPoint extends\n    typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema,\n  CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,\n> = (\n  input: z.infer<EvalDataPoint>,\n  evaluatorOptions?: z.infer<CustomOptions>\n) => Promise<EvalResponse>;\n\n/** An action that evaluates data points and returns evaluation responses. */\nexport type EvaluatorAction<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,\n> = Action<typeof EvalRequestSchema, typeof EvalResponsesSchema> & {\n  __dataPointType?: DataPoint;\n  __configSchema?: CustomOptions;\n};\n\nfunction withMetadata<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,\n>(\n  evaluator: Action<typeof EvalRequestSchema, typeof EvalResponsesSchema>,\n  dataPointType?: DataPoint,\n  configSchema?: CustomOptions\n): EvaluatorAction<DataPoint, CustomOptions> {\n  const withMeta = evaluator as EvaluatorAction<DataPoint, CustomOptions>;\n  withMeta.__dataPointType = dataPointType;\n  withMeta.__configSchema = configSchema;\n  return withMeta;\n}\n\nconst EvalRequestSchema = z.object({\n  dataset: z.array(BaseDataPointSchema),\n  evalRunId: z.string(),\n  options: z.unknown(),\n});\n\n/** Parameters for running an evaluation via {@link evaluate}. */\nexport interface EvaluatorParams<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,\n> {\n  evaluator: EvaluatorArgument<DataPoint, CustomOptions>;\n  dataset: Dataset<DataPoint>;\n  evalRunId?: string;\n  options?: z.infer<CustomOptions>;\n}\n\n/** Configuration options for defining an evaluator via {@link defineEvaluator}. */\nexport interface EvaluatorOptions<\n  DataPoint extends typeof BaseDataPointSchema,\n  EvaluatorOpts extends z.ZodTypeAny,\n> {\n  name: string;\n  displayName: string;\n  definition: string;\n  dataPointType?: DataPoint;\n  configSchema?: EvaluatorOpts;\n  isBilled?: boolean;\n}\n\n/**\n * Creates evaluator action for the provided {@link EvaluatorFn} implementation.\n */\nexport function defineEvaluator<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  EvalDataPoint extends\n    typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema,\n  EvaluatorOpts extends z.ZodTypeAny = z.ZodTypeAny,\n>(\n  registry: Registry,\n  options: EvaluatorOptions<DataPoint, EvaluatorOpts>,\n  runner: EvaluatorFn<EvalDataPoint, EvaluatorOpts>\n): EvaluatorAction {\n  const e = evaluator(options, runner);\n\n  registry.registerAction('evaluator', e);\n\n  return e;\n}\n\n/**\n * Creates evaluator action for the provided {@link EvaluatorFn} implementation.\n */\nexport function evaluator<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  EvalDataPoint extends\n    typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema,\n  EvaluatorOpts extends z.ZodTypeAny = z.ZodTypeAny,\n>(\n  options: EvaluatorOptions<DataPoint, EvaluatorOpts>,\n  runner: EvaluatorFn<EvalDataPoint, EvaluatorOpts>\n): EvaluatorAction {\n  const evalMetadata = {};\n  evalMetadata[EVALUATOR_METADATA_KEY_IS_BILLED] =\n    options.isBilled == undefined ? true : options.isBilled;\n  evalMetadata[EVALUATOR_METADATA_KEY_DISPLAY_NAME] = options.displayName;\n  evalMetadata[EVALUATOR_METADATA_KEY_DEFINITION] = options.definition;\n  if (options.configSchema) {\n    evalMetadata['customOptions'] = toJsonSchema({\n      schema: options.configSchema,\n    });\n  }\n  const evaluator = action(\n    {\n      actionType: 'evaluator',\n      name: options.name,\n      inputSchema: EvalRequestSchema.extend({\n        dataset: options.dataPointType\n          ? z.array(options.dataPointType)\n          : z.array(BaseDataPointSchema),\n        options: options.configSchema ?? z.unknown(),\n        evalRunId: z.string(),\n        batchSize: z.number().optional(),\n      }),\n      outputSchema: EvalResponsesSchema,\n      metadata: {\n        type: 'evaluator',\n        evaluator: evalMetadata,\n      },\n    },\n    async (i) => {\n      const evalResponses: EvalResponses = [];\n      // This also populates missing testCaseIds\n      const batches = getBatchedArray(i.dataset, i.batchSize);\n\n      for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {\n        const batch = batches[batchIndex];\n        try {\n          if (batch.length === 1) {\n            const results = await runBatch(\n              runner,\n              batch,\n              i.batchSize,\n              batchIndex,\n              i.evalRunId,\n              i.options\n            );\n            evalResponses.push(...results);\n          } else {\n            await runInNewSpan(\n              {\n                metadata: {\n                  name: `Batch ${batchIndex}`,\n                  metadata: { 'evaluator:evalRunId': i.evalRunId },\n                },\n              },\n              async (metadata, otSpan) => {\n                const results = await runBatch(\n                  runner,\n                  batch,\n                  i.batchSize,\n                  batchIndex,\n                  i.evalRunId,\n                  i.options,\n                  metadata\n                );\n                evalResponses.push(...results);\n              }\n            );\n          }\n        } catch (e) {\n          logger.error(\n            `Evaluation of batch ${batchIndex} failed: \\n${(e as Error).stack}`\n          );\n          continue;\n        }\n      }\n      return evalResponses;\n    }\n  );\n  const ewm = withMetadata(\n    evaluator as any as Action<\n      typeof EvalRequestSchema,\n      typeof EvalResponsesSchema\n    >,\n    options.dataPointType,\n    options.configSchema\n  );\n  return ewm;\n}\n\n/** Union type for specifying an evaluator: by name string, action, or reference. */\nexport type EvaluatorArgument<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,\n> =\n  | string\n  | EvaluatorAction<DataPoint, CustomOptions>\n  | EvaluatorReference<CustomOptions>;\n\n/**\n * A veneer for interacting with evaluators.\n */\nexport async function evaluate<\n  DataPoint extends typeof BaseDataPointSchema = typeof BaseDataPointSchema,\n  CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,\n>(\n  registry: Registry,\n  params: EvaluatorParams<DataPoint, CustomOptions>\n): Promise<EvalResponses> {\n  let evaluator: EvaluatorAction<DataPoint, CustomOptions>;\n  if (typeof params.evaluator === 'string') {\n    evaluator = await registry.lookupAction(`/evaluator/${params.evaluator}`);\n  } else if (Object.hasOwnProperty.call(params.evaluator, 'info')) {\n    evaluator = await registry.lookupAction(\n      `/evaluator/${params.evaluator.name}`\n    );\n  } else {\n    evaluator = params.evaluator as EvaluatorAction<DataPoint, CustomOptions>;\n  }\n  if (!evaluator) {\n    throw new Error('Unable to utilize the provided evaluator');\n  }\n  return (await evaluator({\n    dataset: params.dataset,\n    options: params.options,\n    evalRunId: params.evalRunId ?? randomUUID(),\n  })) as EvalResponses;\n}\n\n/** Zod schema for evaluator metadata including a label and list of metric names. */\nexport const EvaluatorInfoSchema = z.object({\n  /** Friendly label for this evaluator */\n  label: z.string().optional(),\n  metrics: z.array(z.string()),\n});\nexport type EvaluatorInfo = z.infer<typeof EvaluatorInfoSchema>;\n\n/** A reference to an evaluator, including its name, optional config schema, and info. */\nexport interface EvaluatorReference<CustomOptions extends z.ZodTypeAny> {\n  name: string;\n  configSchema?: CustomOptions;\n  info?: EvaluatorInfo;\n}\n\n/**\n * Helper method to configure a {@link EvaluatorReference} to a plugin.\n */\nexport function evaluatorRef<\n  CustomOptionsSchema extends z.ZodTypeAny = z.ZodTypeAny,\n>(\n  options: EvaluatorReference<CustomOptionsSchema>\n): EvaluatorReference<CustomOptionsSchema> {\n  return { ...options };\n}\n\n/**\n * Helper method to generated batched array. Also ensures each testCase has a\n * testCaseId\n */\nfunction getBatchedArray<T extends { testCaseId?: string }>(\n  arr: T[],\n  batchSize?: number\n): T[][] {\n  let size: number;\n  if (!batchSize) {\n    size = 1;\n  } else {\n    size = batchSize;\n  }\n\n  const batches: T[][] = [];\n  for (var i = 0; i < arr.length; i += size) {\n    batches.push(\n      arr.slice(i, i + size).map((d) => ({\n        ...d,\n        testCaseId: d.testCaseId ?? randomUUID(),\n      }))\n    );\n  }\n\n  return batches;\n}\n\nasync function runBatch<\n  EvalDataPoint extends\n    typeof BaseEvalDataPointSchema = typeof BaseEvalDataPointSchema,\n  EvaluatorOpts extends z.ZodTypeAny = z.ZodTypeAny,\n>(\n  runner: EvaluatorFn<EvalDataPoint, EvaluatorOpts>,\n  batch: BaseDataPoint[],\n  batchSize: number | undefined,\n  batchIndex: number,\n  evalRunId: string,\n  options: any,\n  batchMetadata?: SpanMetadata\n): Promise<EvalResponses> {\n  if (batchMetadata) {\n    batchMetadata.input = batch;\n  }\n  const evalRunPromises = batch.map((d, index) => {\n    const sampleIndex = batchSize ? batchSize * batchIndex + index : batchIndex;\n    const datapoint = d as BaseEvalDataPoint;\n    return runInNewSpan(\n      {\n        metadata: {\n          name: `Test Case ${datapoint.testCaseId}`,\n          metadata: { 'evaluator:evalRunId': evalRunId },\n        },\n      },\n      async (metadata, otSpan) => {\n        const spanId = otSpan.spanContext().spanId;\n        const traceId = otSpan.spanContext().traceId;\n        metadata.input = datapoint;\n        try {\n          const result = await runner(datapoint, options);\n          metadata.output = result;\n          return {\n            ...result,\n            traceId,\n            spanId,\n            sampleIndex,\n          };\n        } catch (error) {\n          return {\n            sampleIndex,\n            spanId,\n            traceId,\n            testCaseId: datapoint.testCaseId,\n            evaluation: {\n              error: `Evaluation of test case ${datapoint.testCaseId} failed: \\n${error}`,\n            },\n          };\n        }\n      }\n    );\n  });\n\n  const allResults = await Promise.all(evalRunPromises);\n  if (batchMetadata) {\n    batchMetadata.output = allResults;\n  }\n  return allResults;\n}\n"],"mappings":"AAgBA,SAAS,QAAQ,SAAsB;AACvC,SAAS,cAAc;AAEvB,SAAS,oBAAoB;AAC7B,SAAuB,oBAAoB;AAC3C,SAAS,kBAAkB;AAEpB,MAAM,cAAc;AACpB,MAAM,kBAAkB,cAAc;AAGtC,MAAM,sBAAsB,EAAE,OAAO;AAAA,EAC1C,OAAO,EAAE,QAAQ;AAAA,EACjB,QAAQ,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC7B,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,SAAS;AAAA,EACvC,WAAW,EAAE,QAAQ,EAAE,SAAS;AAAA,EAChC,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAChC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS;AACzC,CAAC;AAGM,MAAM,0BAA0B,oBAAoB,OAAO;AAAA,EAChE,YAAY,EAAE,OAAO;AACvB,CAAC;AAGD,MAAM,uBAAuB,EAAE,KAAK,CAAC,WAAW,QAAQ,MAAM,CAAC;AAGxD,IAAK,iBAAL,kBAAKA,oBAAL;AACL,EAAAA,gBAAA,aAAU;AACV,EAAAA,gBAAA,UAAO;AACP,EAAAA,gBAAA,UAAO;AAHG,SAAAA;AAAA,GAAA;AAOL,MAAM,cAAc,EAAE,OAAO;AAAA,EAClC,IAAI,EACD,OAAO,EACP;AAAA,IACC;AAAA,EACF,EACC,SAAS;AAAA,EACZ,OAAO,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,CAAC,EAAE,SAAS;AAAA,EAC/D,QAAQ,qBAAqB,SAAS;AAAA,EACtC,OAAO,EAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,SAAS,EACN,OAAO;AAAA,IACN,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,CAAC,EACA,YAAY,EACZ,SAAS;AACd,CAAC;AAGM,MAAM,sCAAsC;AAC5C,MAAM,oCAAoC;AAC1C,MAAM,mCAAmC;AASzC,MAAM,qBAAqB,EAAE,OAAO;AAAA,EACzC,aAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,YAAY,EAAE,OAAO;AAAA,EACrB,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,QAAQ,EAAE,OAAO,EAAE,SAAS;AAAA,EAC5B,YAAY,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,WAAW,CAAC,CAAC;AACzD,CAAC;AAIM,MAAM,sBAAsB,EAAE,MAAM,kBAAkB;AAsB7D,SAAS,aAIPC,YACA,eACA,cAC2C;AAC3C,QAAM,WAAWA;AACjB,WAAS,kBAAkB;AAC3B,WAAS,iBAAiB;AAC1B,SAAO;AACT;AAEA,MAAM,oBAAoB,EAAE,OAAO;AAAA,EACjC,SAAS,EAAE,MAAM,mBAAmB;AAAA,EACpC,WAAW,EAAE,OAAO;AAAA,EACpB,SAAS,EAAE,QAAQ;AACrB,CAAC;AA6BM,SAAS,gBAMd,UACA,SACA,QACiB;AACjB,QAAM,IAAI,UAAU,SAAS,MAAM;AAEnC,WAAS,eAAe,aAAa,CAAC;AAEtC,SAAO;AACT;AAKO,SAAS,UAMd,SACA,QACiB;AACjB,QAAM,eAAe,CAAC;AACtB,eAAa,gCAAgC,IAC3C,QAAQ,YAAY,SAAY,OAAO,QAAQ;AACjD,eAAa,mCAAmC,IAAI,QAAQ;AAC5D,eAAa,iCAAiC,IAAI,QAAQ;AAC1D,MAAI,QAAQ,cAAc;AACxB,iBAAa,eAAe,IAAI,aAAa;AAAA,MAC3C,QAAQ,QAAQ;AAAA,IAClB,CAAC;AAAA,EACH;AACA,QAAMA,aAAY;AAAA,IAChB;AAAA,MACE,YAAY;AAAA,MACZ,MAAM,QAAQ;AAAA,MACd,aAAa,kBAAkB,OAAO;AAAA,QACpC,SAAS,QAAQ,gBACb,EAAE,MAAM,QAAQ,aAAa,IAC7B,EAAE,MAAM,mBAAmB;AAAA,QAC/B,SAAS,QAAQ,gBAAgB,EAAE,QAAQ;AAAA,QAC3C,WAAW,EAAE,OAAO;AAAA,QACpB,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,MACjC,CAAC;AAAA,MACD,cAAc;AAAA,MACd,UAAU;AAAA,QACR,MAAM;AAAA,QACN,WAAW;AAAA,MACb;AAAA,IACF;AAAA,IACA,OAAO,MAAM;AACX,YAAM,gBAA+B,CAAC;AAEtC,YAAM,UAAU,gBAAgB,EAAE,SAAS,EAAE,SAAS;AAEtD,eAAS,aAAa,GAAG,aAAa,QAAQ,QAAQ,cAAc;AAClE,cAAM,QAAQ,QAAQ,UAAU;AAChC,YAAI;AACF,cAAI,MAAM,WAAW,GAAG;AACtB,kBAAM,UAAU,MAAM;AAAA,cACpB;AAAA,cACA;AAAA,cACA,EAAE;AAAA,cACF;AAAA,cACA,EAAE;AAAA,cACF,EAAE;AAAA,YACJ;AACA,0BAAc,KAAK,GAAG,OAAO;AAAA,UAC/B,OAAO;AACL,kBAAM;AAAA,cACJ;AAAA,gBACE,UAAU;AAAA,kBACR,MAAM,SAAS,UAAU;AAAA,kBACzB,UAAU,EAAE,uBAAuB,EAAE,UAAU;AAAA,gBACjD;AAAA,cACF;AAAA,cACA,OAAO,UAAU,WAAW;AAC1B,sBAAM,UAAU,MAAM;AAAA,kBACpB;AAAA,kBACA;AAAA,kBACA,EAAE;AAAA,kBACF;AAAA,kBACA,EAAE;AAAA,kBACF,EAAE;AAAA,kBACF;AAAA,gBACF;AACA,8BAAc,KAAK,GAAG,OAAO;AAAA,cAC/B;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,GAAG;AACV,iBAAO;AAAA,YACL,uBAAuB,UAAU;AAAA,EAAe,EAAY,KAAK;AAAA,UACnE;AACA;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,MAAM;AAAA,IACVA;AAAA,IAIA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV;AACA,SAAO;AACT;AAcA,eAAsB,SAIpB,UACA,QACwB;AACxB,MAAIA;AACJ,MAAI,OAAO,OAAO,cAAc,UAAU;AACxC,IAAAA,aAAY,MAAM,SAAS,aAAa,cAAc,OAAO,SAAS,EAAE;AAAA,EAC1E,WAAW,OAAO,eAAe,KAAK,OAAO,WAAW,MAAM,GAAG;AAC/D,IAAAA,aAAY,MAAM,SAAS;AAAA,MACzB,cAAc,OAAO,UAAU,IAAI;AAAA,IACrC;AAAA,EACF,OAAO;AACL,IAAAA,aAAY,OAAO;AAAA,EACrB;AACA,MAAI,CAACA,YAAW;AACd,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AACA,SAAQ,MAAMA,WAAU;AAAA,IACtB,SAAS,OAAO;AAAA,IAChB,SAAS,OAAO;AAAA,IAChB,WAAW,OAAO,aAAa,WAAW;AAAA,EAC5C,CAAC;AACH;AAGO,MAAM,sBAAsB,EAAE,OAAO;AAAA;AAAA,EAE1C,OAAO,EAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC;AAC7B,CAAC;AAaM,SAAS,aAGd,SACyC;AACzC,SAAO,EAAE,GAAG,QAAQ;AACtB;AAMA,SAAS,gBACP,KACA,WACO;AACP,MAAI;AACJ,MAAI,CAAC,WAAW;AACd,WAAO;AAAA,EACT,OAAO;AACL,WAAO;AAAA,EACT;AAEA,QAAM,UAAiB,CAAC;AACxB,WAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,MAAM;AACzC,YAAQ;AAAA,MACN,IAAI,MAAM,GAAG,IAAI,IAAI,EAAE,IAAI,CAAC,OAAO;AAAA,QACjC,GAAG;AAAA,QACH,YAAY,EAAE,cAAc,WAAW;AAAA,MACzC,EAAE;AAAA,IACJ;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAe,SAKb,QACA,OACA,WACA,YACA,WACA,SACA,eACwB;AACxB,MAAI,eAAe;AACjB,kBAAc,QAAQ;AAAA,EACxB;AACA,QAAM,kBAAkB,MAAM,IAAI,CAAC,GAAG,UAAU;AAC9C,UAAM,cAAc,YAAY,YAAY,aAAa,QAAQ;AACjE,UAAM,YAAY;AAClB,WAAO;AAAA,MACL;AAAA,QACE,UAAU;AAAA,UACR,MAAM,aAAa,UAAU,UAAU;AAAA,UACvC,UAAU,EAAE,uBAAuB,UAAU;AAAA,QAC/C;AAAA,MACF;AAAA,MACA,OAAO,UAAU,WAAW;AAC1B,cAAM,SAAS,OAAO,YAAY,EAAE;AACpC,cAAM,UAAU,OAAO,YAAY,EAAE;AACrC,iBAAS,QAAQ;AACjB,YAAI;AACF,gBAAM,SAAS,MAAM,OAAO,WAAW,OAAO;AAC9C,mBAAS,SAAS;AAClB,iBAAO;AAAA,YACL,GAAG;AAAA,YACH;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd,iBAAO;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA,YAAY,UAAU;AAAA,YACtB,YAAY;AAAA,cACV,OAAO,2BAA2B,UAAU,UAAU;AAAA,EAAc,KAAK;AAAA,YAC3E;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AAED,QAAM,aAAa,MAAM,QAAQ,IAAI,eAAe;AACpD,MAAI,eAAe;AACjB,kBAAc,SAAS;AAAA,EACzB;AACA,SAAO;AACT;","names":["EvalStatusEnum","evaluator"]}