eve/dist/src/evals/cli/eval.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{randomUUID}from"node:crypto";import{readFile}from"node:fs/promises";import{loadDevelopmentEnvironmentFiles}from"#cli/dev/environment.js";import{basename,join}from"node:path";import{createDevelopmentServer}from"#internal/nitro/host.js";import{EVE_EVALUATION_ENV_FLAG,EVE_EVALUATION_RUN_ID_ENV}from"#internal/application/dev-environment.js";import{shutdownActiveSandboxHandles}from"#execution/sandbox/active-handles.js";import{createEvalClient}from"#evals/cli/eval-client.js";import{filterEvalsByTags}from"#evals/cli/filter.js";import{discoverAndImportEvals,discoverEvalConfig,findMisplacedEvalDirs}from"#evals/runner/discover.js";import{runEvals}from"#evals/runner/run-evals.js";import{Console}from"#evals/runner/reporters/console.js";import{JUnit}from"#evals/runner/reporters/junit.js";import{resolveEvalTargetHandle}from"#evals/target.js";async function runEvalCommand(e,t,n,r=process.cwd()){await loadDevelopmentEnvironmentFiles(r);let i=e.length>0?e:void 0,a=await discoverAndImportEvals(r,i);if(a.length===0){let e=await findMisplacedEvalDirs(r);e.length>0?n.error(`No evals found under evals/, but eval files are present inside agent/:
`+e.map(e=>`  - ${e}`).join(`
`)+`
eve eval only scans the top-level evals/ directory (a sibling of agent/). Move these files there.`):i?n.error(`No evals found matching: ${i.join(`, `)}`):n.error(`No evals found. Create files under evals/ with the *.eval.ts extension.`),process.exitCode=2;return}let o=t.tag??[],s=t.excludeTag??[],c=filterEvalsByTags({evaluations:a,includeTags:o,excludeTags:[]});if(c.length===0){n.error(`No evals matched the provided tags (${o.join(`, `)}).`),process.exitCode=2;return}let l=filterEvalsByTags({evaluations:c,includeTags:[],excludeTags:s});if(t.list===!0){printEvalList(l,t.json===!0,n);return}if(l.length===0){n.log(`All ${c.length} matching evals are excluded by tags (${s.join(`, `)}); nothing to run.`);return}let u,d;try{u=parsePositiveInteger(t.maxConcurrency,`--max-concurrency`),d=parseNonNegativeInteger(t.timeout,`--timeout`)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let f;try{f=await discoverEvalConfig(r)}catch(e){n.error(e instanceof Error?e.message:String(e)),process.exitCode=2;return}let p,m,h;try{if(t.url)h=await createEvalClient({kind:`remote`,url:t.url},{workspaceRoot:r}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(r),kind:`remote`,url:t.url});else{process.env[EVE_EVALUATION_ENV_FLAG]=`1`,process.env[EVE_EVALUATION_RUN_ID_ENV]=randomUUID(),p=createDevelopmentServer(r,{host:`127.0.0.1`,port:0});let e=await p.start();h=await createEvalClient({kind:`local`,url:e.url}),m=await resolveEvalTargetHandle({client:h,expectedAgentName:await readExpectedAgentName(r),kind:`local`,url:e.url})}let e=t.json===!0?[]:[Console()];t.junit!==void 0&&e.push(JUnit({filePath:t.junit}));let i=await runEvals({evaluations:l,config:f,target:m,client:h,appRoot:r,reporters:e,includeEvalReporters:t.skipReport!==!0,maxConcurrency:u,timeoutMs:d,onEvalLog:t.verbose===!0?(e,t)=>n.log(`[${e}] ${t}`):void 0});t.json&&n.log(JSON.stringify(i,null,2));let a=i.failed>0,o=t.strict===!0&&i.scored>0;(a||o)&&(process.exitCode=1)}finally{p&&(await p.close(),await shutdownActiveSandboxHandles({log:e=>n.error(e)}))}let g=typeof process.exitCode==`number`?process.exitCode:0;await flushStandardStreams(),process.exit(g)}async function flushStandardStreams(){await Promise.all([flushStream(process.stdout),flushStream(process.stderr)])}async function flushStream(e){await new Promise(t=>{let n=!1,settle=()=>{n||(n=!0,setImmediate(()=>{e.off(`error`,settle),t()}))};e.once(`error`,settle);try{e.write(``,settle)}catch{settle()}})}function parsePositiveInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<1)throw Error(`${t} must be a positive integer; got "${e}".`);return n}function parseNonNegativeInteger(e,t){if(e===void 0)return;let n=Number(e);if(!Number.isInteger(n)||n<0)throw Error(`${t} must be a non-negative integer; got "${e}".`);return n}function printEvalList(e,t,n){if(t){let t=e.map(e=>({id:e.id,description:e.description,tags:e.tags}));n.log(JSON.stringify(t,null,2));return}for(let t of e){let e=t.description===void 0?``:` — ${t.description}`,r=t.tags!==void 0&&t.tags.length>0?` [${t.tags.join(`, `)}]`:``;n.log(`${t.id}${r}${e}`)}}async function readExpectedAgentName(e){try{let t=JSON.parse(await readFile(join(e,`package.json`),`utf8`));return typeof t.name==`string`&&t.name.length>0?t.name:basename(e)}catch{return basename(e)}}export{runEvalCommand};