eve/dist/src/evals/assertions/run.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{deepEquals,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function completed(){return{name:`completed`,evaluate(e){return e.status===`failed`?fail(failureDetail(`run failed`,e.derived.failureCode)):e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS}}}function waiting(){return{name:`waiting`,evaluate(e){return e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`)}}}function didNotFail(){return{name:`didNotFail`,evaluate(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?PASS:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let r=joinCompletedMessages(t.events);return(typeof e==`string`?r.includes(e):testRegExp(e,r))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(r)}`)}}}function calledTool(e,t={}){return{name:`calledTool(${e})`,evaluate(n){let i=n.derived.toolCalls.filter(t=>t.name===e),a=i.filter(e=>toolCallMatches(e,t));if(t.times===void 0?a.length>0:a.length===t.times)return{score:1,metadata:{matchingCalls:a.length}};let o=i.length>0?`observed ${e} calls: ${i.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`;return fail(`${t.times===void 0?`expected a matching call to "${e}"`:`expected exactly ${t.times} matching call(s), found ${a.length}`}; ${o}`)}}}function loadedSkill(e,t={}){return{...calledTool(LOAD_SKILL_TOOL_NAME,{...t,input:{skill:e}}),name:`loadedSkill(${e})`}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=t.derived.toolCalls.map(e=>e.name),r=0;for(let t of n)if(t===e[r]&&(r+=1),r===e.length)break;return r===e.length?PASS:fail(`missing "${e[r]}" after [${e.slice(0,r).join(`, `)}]; observed order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(e=>e.data.result.kind===`tool-result`?e.data.result.toolName:e.data.result.kind);return fail(`${t.length} failed action(s): ${n.join(`, `)}`)}}}function calledSubagent(e,n={}){return{name:`calledSubagent(${e})`,evaluate(r){let i=r.derived.subagentCalls.filter(t=>t.name===e);return i.filter(e=>subagentCallMatches(e,n)).length>0?PASS:i.length===0?fail(`subagent "${e}" was never called; observed: [${r.derived.subagentCalls.map(e=>e.name).join(`, `)}]`,{observedSubagentCalls:r.derived.subagentCalls}):fail(`subagent "${e}" was called but no call matched the constraints`,{observedSubagentCalls:i})}}}function event(e,t){return{name:`event(${t})`,evaluate(n){return e(n.events)?PASS:fail(`event predicate "${t}" did not hold`)}}}function outputEquals(t){return{name:`outputEquals`,evaluate(n){return deepEquals(n.output,t)?PASS:fail(`output ${truncate(JSON.stringify(n.output))} does not equal expected ${truncate(JSON.stringify(t))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);return!(`issues`in n)||n.issues===void 0?PASS:fail(`output failed schema validation: ${n.issues.map(e=>e.message).join(`; `)}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&n.data.message!==null&&t.push(n.data.message);return t.join(`
`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}export{calledSubagent,calledTool,completed,didNotFail,event,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,outputEquals,outputMatches,toolOrder,usedNoTools,waiting};