@genkit-ai/ai/lib/testing/model-tester.mjs.map

Genkit AI framework generative AI APIs.

{"version":3,"sources":["../../src/testing/model-tester.ts"],"sourcesContent":["/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { z } from '@genkit-ai/core';\nimport { Registry } from '@genkit-ai/core/registry';\nimport { runInNewSpan } from '@genkit-ai/core/tracing';\nimport * as assert from 'assert';\nimport { generate } from '../generate';\nimport { ModelAction } from '../model';\nimport { defineTool } from '../tool';\n\nconst tests: Record<string, TestCase> = {\n  'basic hi': async (registry: Registry, model: string) => {\n    const response = await generate(registry, {\n      model,\n      prompt: 'just say \"Hi\", literally',\n    });\n\n    const got = response.text.trim();\n    assert.match(got, /Hi/i);\n  },\n  multimodal: async (registry: Registry, model: string) => {\n    const resolvedModel = (await registry.lookupAction(\n      `/model/${model}`\n    )) as ModelAction;\n    if (!resolvedModel.__action.metadata?.model.supports?.media) {\n      skip();\n    }\n    const response = await generate(registry, {\n      model,\n      prompt: [\n        {\n          media: {\n            url: 'data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAABhGlDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV9TpSoVETOIOGSoulgQFXHUKhShQqgVWnUwufRDaNKQtLg4Cq4FBz8Wqw4uzro6uAqC4AeIs4OToouU+L+k0CLGg+N+vLv3uHsHCLUi0+22MUA3ylYyHpPSmRUp9IpOhCCiFyMKs81ZWU7Ad3zdI8DXuyjP8j/35+jWsjYDAhLxDDOtMvE68dRm2eS8TyyygqIRnxOPWnRB4keuqx6/cc67LPBM0Uol54hFYinfwmoLs4KlE08SRzTdoHwh7bHGeYuzXqywxj35C8NZY3mJ6zQHEccCFiFDgooKNlBEGVFaDVJsJGk/5uMfcP0yuVRybYCRYx4l6FBcP/gf/O7Wzk2Me0nhGND+4jgfQ0BoF6hXHef72HHqJ0DwGbgymv5SDZj+JL3a1CJHQM82cHHd1NQ94HIH6H8yFUtxpSBNIZcD3s/omzJA3y3Qter11tjH6QOQoq4SN8DBITCcp+w1n3d3tPb275lGfz9aC3Kd0jYiSQAAAAlwSFlzAAAuIwAALiMBeKU/dgAAAAd0SU1FB+gJBxQRO1/5qB8AAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAAsUlEQVQoz61SMQqEMBDcO5SYToUE/IBPyRMCftAH+INUviApUwYjNkKCVcTiQK7IHSw45czODrMswCOQUkopEQZjzDiOWemdZfu+b5oGYYgx1nWNMPwB2vACAK01Y4wQ8qGqqirL8jzPlNI9t64r55wQUgBA27be+xDCfaJhGJxzSqnv3UKIn7ne+2VZEB2stZRSRLN93+d5RiRs28Y5RySEEI7jyEpFlp2mqeu6Zx75ApQwPdsIcq0ZAAAAAElFTkSuQmCC',\n          },\n        },\n        {\n          text: 'what math operation is this? plus, minus, multiply or divide?',\n        },\n      ],\n    });\n\n    const want = /plus/i;\n    const got = response.text.trim();\n    assert.match(got, want);\n  },\n  history: async (registry: Registry, model: string) => {\n    const resolvedModel = (await registry.lookupAction(\n      `/model/${model}`\n    )) as ModelAction;\n    if (!resolvedModel.__action.metadata?.model.supports?.multiturn) {\n      skip();\n    }\n    const response1 = await generate(registry, {\n      model,\n      prompt: 'My name is Glorb',\n    });\n    const response = await generate(registry, {\n      model,\n      prompt: \"What's my name?\",\n      messages: response1.messages,\n    });\n\n    const got = response.text.trim();\n    assert.match(got, /Glorb/);\n  },\n  'system prompt': async (registry: Registry, model: string) => {\n    const { text } = await generate(registry, {\n      model,\n      prompt: 'Hi',\n      messages: [\n        {\n          role: 'system',\n          content: [\n            {\n              text: 'If the user says \"Hi\", just say \"Bye\" ',\n            },\n          ],\n        },\n      ],\n    });\n\n    const want = 'Bye';\n    const got = text.trim();\n    assert.equal(got, want);\n  },\n  'structured output': async (registry: Registry, model: string) => {\n    const response = await generate(registry, {\n      model,\n      prompt: 'extract data as json from: Jack was a Lumberjack',\n      output: {\n        format: 'json',\n        schema: z.object({\n          name: z.string(),\n          occupation: z.string(),\n        }),\n      },\n    });\n\n    const want = {\n      name: 'Jack',\n      occupation: 'Lumberjack',\n    };\n    const got = response.output;\n    assert.deepEqual(want, got);\n  },\n  'tool calling': async (registry: Registry, model: string) => {\n    const resolvedModel = (await registry.lookupAction(\n      `/model/${model}`\n    )) as ModelAction;\n    if (!resolvedModel.__action.metadata?.model.supports?.tools) {\n      skip();\n    }\n\n    const { text } = await generate(registry, {\n      model,\n      prompt: 'what is a gablorken of 2? use provided tool',\n      tools: ['gablorkenTool'],\n    });\n\n    const got = text.trim();\n    assert.match(got, /9.407/);\n  },\n};\n\ntype TestReport = {\n  description: string;\n  models: {\n    name: string;\n    passed: boolean;\n    skipped?: boolean;\n    error?: {\n      message: string;\n      stack?: string;\n    };\n  }[];\n}[];\n\ntype TestCase = (ai: Registry, model: string) => Promise<void>;\n\nexport async function testModels(\n  registry: Registry,\n  models: string[]\n): Promise<TestReport> {\n  defineTool(\n    registry,\n    {\n      name: 'gablorkenTool',\n      description: 'use when need to calculate a gablorken',\n      inputSchema: z.object({\n        value: z.number(),\n      }),\n      outputSchema: z.number(),\n    },\n    async (input) => {\n      return Math.pow(input.value, 3) + 1.407;\n    }\n  );\n\n  return await runInNewSpan(\n    registry,\n    { metadata: { name: 'testModels' } },\n    async () => {\n      const report: TestReport = [];\n      for (const test of Object.keys(tests)) {\n        await runInNewSpan(registry, { metadata: { name: test } }, async () => {\n          report.push({\n            description: test,\n            models: [],\n          });\n          const caseReport = report[report.length - 1];\n          for (const model of models) {\n            caseReport.models.push({\n              name: model,\n              passed: true, // optimistically\n            });\n            const modelReport = caseReport.models[caseReport.models.length - 1];\n            try {\n              await tests[test](registry, model);\n            } catch (e) {\n              modelReport.passed = false;\n              if (e instanceof SkipTestError) {\n                modelReport.skipped = true;\n              } else if (e instanceof Error) {\n                modelReport.error = {\n                  message: e.message,\n                  stack: e.stack,\n                };\n              } else {\n                modelReport.error = {\n                  message: `${e}`,\n                };\n              }\n            }\n          }\n        });\n      }\n\n      return report;\n    }\n  );\n}\n\nclass SkipTestError extends Error {}\n\nfunction skip() {\n  throw new SkipTestError();\n}\n"],"mappings":"AAgBA,SAAS,SAAS;AAElB,SAAS,oBAAoB;AAC7B,YAAY,YAAY;AACxB,SAAS,gBAAgB;AAEzB,SAAS,kBAAkB;AAE3B,MAAM,QAAkC;AAAA,EACtC,YAAY,OAAO,UAAoB,UAAkB;AACvD,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,IACV,CAAC;AAED,UAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,WAAO,MAAM,KAAK,KAAK;AAAA,EACzB;AAAA,EACA,YAAY,OAAO,UAAoB,UAAkB;AACvD,UAAM,gBAAiB,MAAM,SAAS;AAAA,MACpC,UAAU,KAAK;AAAA,IACjB;AACA,QAAI,CAAC,cAAc,SAAS,UAAU,MAAM,UAAU,OAAO;AAC3D,WAAK;AAAA,IACP;AACA,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,QACN;AAAA,UACE,OAAO;AAAA,YACL,KAAK;AAAA,UACP;AAAA,QACF;AAAA,QACA;AAAA,UACE,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,OAAO;AACb,UAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB;AAAA,EACA,SAAS,OAAO,UAAoB,UAAkB;AACpD,UAAM,gBAAiB,MAAM,SAAS;AAAA,MACpC,UAAU,KAAK;AAAA,IACjB;AACA,QAAI,CAAC,cAAc,SAAS,UAAU,MAAM,UAAU,WAAW;AAC/D,WAAK;AAAA,IACP;AACA,UAAM,YAAY,MAAM,SAAS,UAAU;AAAA,MACzC;AAAA,MACA,QAAQ;AAAA,IACV,CAAC;AACD,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,UAAU,UAAU;AAAA,IACtB,CAAC;AAED,UAAM,MAAM,SAAS,KAAK,KAAK;AAC/B,WAAO,MAAM,KAAK,OAAO;AAAA,EAC3B;AAAA,EACA,iBAAiB,OAAO,UAAoB,UAAkB;AAC5D,UAAM,EAAE,KAAK,IAAI,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS;AAAA,YACP;AAAA,cACE,MAAM;AAAA,YACR;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,OAAO;AACb,UAAM,MAAM,KAAK,KAAK;AACtB,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB;AAAA,EACA,qBAAqB,OAAO,UAAoB,UAAkB;AAChE,UAAM,WAAW,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ,EAAE,OAAO;AAAA,UACf,MAAM,EAAE,OAAO;AAAA,UACf,YAAY,EAAE,OAAO;AAAA,QACvB,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,UAAM,OAAO;AAAA,MACX,MAAM;AAAA,MACN,YAAY;AAAA,IACd;AACA,UAAM,MAAM,SAAS;AACrB,WAAO,UAAU,MAAM,GAAG;AAAA,EAC5B;AAAA,EACA,gBAAgB,OAAO,UAAoB,UAAkB;AAC3D,UAAM,gBAAiB,MAAM,SAAS;AAAA,MACpC,UAAU,KAAK;AAAA,IACjB;AACA,QAAI,CAAC,cAAc,SAAS,UAAU,MAAM,UAAU,OAAO;AAC3D,WAAK;AAAA,IACP;AAEA,UAAM,EAAE,KAAK,IAAI,MAAM,SAAS,UAAU;AAAA,MACxC;AAAA,MACA,QAAQ;AAAA,MACR,OAAO,CAAC,eAAe;AAAA,IACzB,CAAC;AAED,UAAM,MAAM,KAAK,KAAK;AACtB,WAAO,MAAM,KAAK,OAAO;AAAA,EAC3B;AACF;AAiBA,eAAsB,WACpB,UACA,QACqB;AACrB;AAAA,IACE;AAAA,IACA;AAAA,MACE,MAAM;AAAA,MACN,aAAa;AAAA,MACb,aAAa,EAAE,OAAO;AAAA,QACpB,OAAO,EAAE,OAAO;AAAA,MAClB,CAAC;AAAA,MACD,cAAc,EAAE,OAAO;AAAA,IACzB;AAAA,IACA,OAAO,UAAU;AACf,aAAO,KAAK,IAAI,MAAM,OAAO,CAAC,IAAI;AAAA,IACpC;AAAA,EACF;AAEA,SAAO,MAAM;AAAA,IACX;AAAA,IACA,EAAE,UAAU,EAAE,MAAM,aAAa,EAAE;AAAA,IACnC,YAAY;AACV,YAAM,SAAqB,CAAC;AAC5B,iBAAW,QAAQ,OAAO,KAAK,KAAK,GAAG;AACrC,cAAM,aAAa,UAAU,EAAE,UAAU,EAAE,MAAM,KAAK,EAAE,GAAG,YAAY;AACrE,iBAAO,KAAK;AAAA,YACV,aAAa;AAAA,YACb,QAAQ,CAAC;AAAA,UACX,CAAC;AACD,gBAAM,aAAa,OAAO,OAAO,SAAS,CAAC;AAC3C,qBAAW,SAAS,QAAQ;AAC1B,uBAAW,OAAO,KAAK;AAAA,cACrB,MAAM;AAAA,cACN,QAAQ;AAAA;AAAA,YACV,CAAC;AACD,kBAAM,cAAc,WAAW,OAAO,WAAW,OAAO,SAAS,CAAC;AAClE,gBAAI;AACF,oBAAM,MAAM,IAAI,EAAE,UAAU,KAAK;AAAA,YACnC,SAAS,GAAG;AACV,0BAAY,SAAS;AACrB,kBAAI,aAAa,eAAe;AAC9B,4BAAY,UAAU;AAAA,cACxB,WAAW,aAAa,OAAO;AAC7B,4BAAY,QAAQ;AAAA,kBAClB,SAAS,EAAE;AAAA,kBACX,OAAO,EAAE;AAAA,gBACX;AAAA,cACF,OAAO;AACL,4BAAY,QAAQ;AAAA,kBAClB,SAAS,GAAG,CAAC;AAAA,gBACf;AAAA,cACF;AAAA,YACF;AAAA,UACF;AAAA,QACF,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,MAAM,sBAAsB,MAAM;AAAC;AAEnC,SAAS,OAAO;AACd,QAAM,IAAI,cAAc;AAC1B;","names":[]}