eve/dist/src/evals/cli/eval.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{startDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{ConsoleReporter}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let o=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(o);let s=e.length>0?e:void 0,c=await discoverAndImportEvals(o,s);if(c.length===0){s?n.error(`No evals found matching: ${s.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let l=filterEvalsByTag(c,t.tag??[]);if(l.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(l,t.json===!0,n);return}let f;try{f=await discoverEvalConfig(o)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m,h;try{t.url?(h=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:o}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`remote`,url:t.url})):(p=await startDevelopmentServer(o,{host:`127.0.0.1`,port:0}),h=await createEvalClient({kind:`local`,url:p.url}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`local`,url:p.url}));let e=t.json===!0?[]:[new ConsoleReporter];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let r=await runEvals({evaluations:l,config:f,target:m,client:h,appRoot:o,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(r,null,2));let i=r.failed>0,a=t.strict===!0&&r.scored>0;(i||a)&&(process.exitCode=1)}finally{p&&await p.close()}let g=typeof process.exitCode==`number`?process.exitCode:0;process.exit(g)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};