UNPKG

eve

Version:

Filesystem-first framework for durable backend AI agents that run anywhere.

github.com/vercel/eve

vercel/eve

1 lines • 3.3 kB
1
import{resolveArtifactDirectory,writeArtifacts}from"#evals/runner/artifacts.js";import{executeEval}from"#evals/runner/execute-eval.js";async function runEvals(e){let{config:t,target:n,client:r,appRoot:i}=e,a=e.maxConcurrency??t.maxConcurrency??8;if(!Number.isInteger(a)||a<1)throw Error(`Eval maxConcurrency must be a positive integer; got ${String(e.maxConcurrency??t.maxConcurrency)}.`);let o=e.evaluations.map(e=>applyConfigDefaults(e,t)),s=new Date().toISOString(),c=buildReporterBindings({...e,evaluations:o});for(let e of c)await e.reporter.onRunStart(o.filter(t=>e.evalIds.has(t.id)),n);let l=[],u=[...o],d=new Set,f=Promise.resolve(),enqueueReporterCallback=(e,t)=>{f=f.then(async()=>{for(let n of c)n.evalIds.has(e.id)&&await t(n.reporter)})};for(;u.length>0||d.size>0;){for(;u.length>0&&d.size<a;){let t=u.shift();if(t===void 0)break;let i=new Date().toISOString();enqueueReporterCallback(t,async e=>{await e.onEvalStart?.({evaluation:t,startedAt:i,target:n})});let a=(async()=>{let a=await executeEval({client:r,evaluation:t,setupContext:e.setupContext,onLog:e.onEvalLog===void 0?void 0:n=>e.onEvalLog?.(t.id,n),onSessionStart:e=>{enqueueReporterCallback(t,async r=>{await r.onSessionStart?.({evaluation:t,primary:e.primary,sessionId:e.sessionId,startedAt:e.startedAt,target:n,traceContext:e.traceContext})})},startedAt:i,target:n,timeoutMs:e.timeoutMs});l.push(a);for(let n of a.result.sessions??[]){if(n.sessionId===void 0)continue;let r=n.primary?`primary session`:`secondary session`;e.onEvalLog?.(t.id,`workflow run id (${r}): ${n.sessionId}`)}enqueueReporterCallback(t,async e=>{await e.onEvalComplete(a,{evaluation:t,target:n,traceContexts:a.result.traceContexts})})})().finally(()=>{d.delete(a)});d.add(a)}d.size>0&&await Promise.race(d)}await f;let p=new Map(o.map((e,t)=>[e.id,t]));l.sort((e,t)=>(p.get(e.id)??0)-(p.get(t.id)??0));let m=buildSummary(n,l,s),h=resolveArtifactDirectory(i);await writeArtifacts(h,m);for(let e of c)await e.reporter.onRunComplete(scopeSummary(m,e.evalIds));return m}function buildReporterBindings(e){let t=new Set(e.evaluations.map(e=>e.id)),n=new Set(e.reporters);if(e.includeEvalReporters!==!1)for(let t of e.config.reporters??[])n.add(t);let r=[...n].map(e=>({reporter:e,evalIds:t}));if(e.includeEvalReporters===!1)return r;let i=new Map;for(let t of e.evaluations)for(let e of t.reporters??[]){if(n.has(e))continue;let r=i.get(e)??new Set;r.add(t.id),i.set(e,r)}for(let[e,t]of i)r.push({reporter:e,evalIds:t});return r}function applyConfigDefaults(e,t){let n=e.judge??t.judge,r=e.timeoutMs??t.timeoutMs;return n===e.judge&&r===e.timeoutMs?e:{...e,judge:n,timeoutMs:r}}function buildSummary(e,t,n){return{target:e,results:t,startedAt:n,completedAt:new Date().toISOString(),passed:countVerdicts(t,`passed`),failed:countVerdicts(t,`failed`),scored:countVerdicts(t,`scored`),skipped:countVerdicts(t,`skipped`),errored:t.filter(e=>e.error!==void 0).length}}function scopeSummary(e,t){if(e.results.every(e=>t.has(e.id)))return e;let n=e.results.filter(e=>t.has(e.id));return{...e,results:n,passed:countVerdicts(n,`passed`),failed:countVerdicts(n,`failed`),scored:countVerdicts(n,`scored`),skipped:countVerdicts(n,`skipped`),errored:n.filter(e=>e.error!==void 0).length}}function countVerdicts(e,t){return e.filter(e=>e.verdict===t).length}export{runEvals};