eve/dist/src/evals/cli/eval.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{basename,join}from"node:path";import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{resolveApplicationRoot}from"#internal/application/paths.js";import{shutdownActiveSandboxHandles}from"#execution/sandbox/active-handles.js";import{createDevelopmentServer}from"#internal/nitro/host.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n){let o=resolveApplicationRoot();loadDevelopmentEnvironmentFiles(o);let s=e.length>0?e:void 0,c=await discoverAndImportEvals(o,s);if(c.length===0){let e=await findMisplacedEvalDirs(o);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
`+e.map(e=>`  - ${e}`).join(`
`)+`
eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):s?n.error(`No evals found matching: ${s.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let l=filterEvalsByTag(c,t.tag??[]);if(l.length===0){n.error(`No evals matched the provided tags (${(t.tag??[]).join(`, `)}).`),process.exitCode=2;return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}if(t.list===!0){printEvalList(l,t.json===!0,n);return}let f;try{f=await discoverEvalConfig(o)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m,h;try{if(t.url)h=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:o}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`remote`,url:t.url});else{p=createDevelopmentServer(o,{host:`127.0.0.1`,port:0});let e=await p.start();h=await createEvalClient({kind:`local`,url:e.url}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(o),kind:`local`,url:e.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let r=await runEvals({evaluations:l,config:f,target:m,client:h,appRoot:o,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(r,null,2));let i=r.failed>0,a=t.strict===!0&&r.scored>0;(i||a)&&(process.exitCode=1)}finally{p&&(await p.close(),await shutdownActiveSandboxHandles({log:e=>n.error(e)}))}let g=typeof process.exitCode==`number`?process.exitCode:0;process.exit(g)}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function filterEvalsByTag(e,t){return t.length===0?[...e]:e.filter(e=>e.tags?.some(e=>t.includes(e))??!1)}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(r){try{let i=JSON.parse(await readFile(join(r,`package.json`),`utf8`));return typeof i.name==`string`&&i.name.length>0?i.name:basename(r)}catch{return basename(r)}}export{runEvalCommand};