eve/dist/src/evals/assertions/run.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{deepEquals,eventMatches,subagentCallMatches,testRegExp,toolCallMatches}from"#evals/match.js";import{LOAD_SKILL_TOOL_NAME}from"#runtime/skills/fragment-context.js";const PASS={score:1},fail=(e,t)=>({score:0,message:e,metadata:t});function succeeded(){return{name:`succeeded`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?fail(`run parked on ${e.derived.inputRequests.length} unanswered input request(s)`):PASS:t}}}function parked(){return{name:`parked`,evaluate(e){let t=runFailure(e);return t===void 0?e.derived.parked?PASS:fail(`expected the run to park on HITL input; it ended "${e.status}" with no pending requests`):t}}}function messageIncludes(e){return{name:`messageIncludes(${String(e)})`,evaluate(t){let n=joinCompletedMessages(t.events);return(typeof e==`string`?n.includes(e):testRegExp(e,n))?PASS:fail(`assistant messages did not include ${String(e)}; got: ${truncate(n)}`)}}}function calledTool(e,t={}){return validateCount(t.count),{name:`calledTool(${e})`,evaluate(n){let r=n.derived.toolCalls.filter(t=>t.name===e),a=r.filter(e=>toolCallMatches(e,t));if(matchesCount(t.count,a.length))return{score:1,metadata:{matchingCalls:a.length}};let o=r.length>0?`observed ${e} calls: ${r.map(e=>truncate(JSON.stringify(e.input))).join(`, `)}`:`observed tools: [${n.derived.toolCalls.map(e=>e.name).join(`, `)}]`,s;return s=t.count===void 0?`expected a matching call to "${e}"`:typeof t.count==`number`?`expected exactly ${t.count} matching call(s), found ${a.length}`:`expected matching call count to satisfy the count predicate, found ${a.length}`,fail(`${s}; ${o}`)}}}function loadedSkill(e,t={}){return{...calledTool(LOAD_SKILL_TOOL_NAME,{...t,input:{skill:e}}),name:`loadedSkill(${e})`}}function notCalledTool(e){return{name:`notCalledTool(${e})`,evaluate(t){let n=t.derived.toolCalls.filter(t=>t.name===e).length;return n===0?PASS:fail(`"${e}" was called ${n} time(s)`)}}}function toolOrder(e){return{name:`toolOrder(${e.join(` → `)})`,evaluate(t){let n=requestedTools(t.events).map(e=>e.name),r=missingOrderedName(e,n);return r===void 0?PASS:fail(`missing "${r.name}" after [${e.slice(0,r.cursor).join(`, `)}]; observed request order: [${n.join(`, `)}]`)}}}function usedNoTools(){return{name:`usedNoTools`,evaluate(e){let t=e.derived.toolCallCount;return t===0?PASS:fail(`expected no tool calls, got ${t}`,{toolCallCount:t})}}}function maxToolCalls(e){return validateCount(e),{name:`maxToolCalls(${e})`,evaluate(t){let n=t.derived.toolCallCount;return n<=e?PASS:fail(`expected at most ${e} tool calls, got ${n}`,{maxAllowed:e,toolCallCount:n})}}}function noFailedActions(){return{name:`noFailedActions`,evaluate(e){let t=e.events.filter(e=>e.type===`action.result`&&(e.data.status===`failed`||e.data.result.isError===!0));if(t.length===0)return PASS;let n=t.map(formatFailedActionResult);return fail(`${t.length} failed action(s): ${n.join(`; `)}`,{failedActions:t.map(e=>({callId:e.data.result.callId,error:e.data.error,kind:e.data.result.kind,output:e.data.result.output,status:e.data.status}))})}}}function calledSubagent(e,t={}){return validateCount(t.count),{name:`calledSubagent(${e})`,evaluate(r){let i=r.derived.subagentCalls.filter(t=>t.name===e),a=i.filter(e=>subagentCallMatches(e,t));return matchesCount(t.count,a.length)?{score:1,metadata:{matchingCalls:a.length}}:i.length===0?fail(`subagent "${e}" was never called; observed: [${r.derived.subagentCalls.map(e=>e.name).join(`, `)}]`,{observedSubagentCalls:r.derived.subagentCalls}):fail(`subagent "${e}" was called but ${a.length} call(s) matched the constraints`,{observedSubagentCalls:i})}}}function eventsSatisfy(e,t){return{name:`eventsSatisfy(${e})`,evaluate(n){return t(n.events)?PASS:fail(`event predicate "${e}" did not hold`)}}}function typedEvent(e){return validateCount(e.count),{name:`event(${e.type})`,evaluate(n){let r=n.events.filter(n=>eventMatches(n,e));return matchesCount(e.count,r.length)?{score:1,metadata:{matchingEvents:r.length}}:fail(`expected ${describeCount(e.count)} matching ${e.type} event(s), found ${r.length}; observed: [${n.events.map(e=>e.type).join(`, `)}]`)}}}function notEvent(e){return{name:`notEvent(${e.type})`,evaluate(n){let r=n.events.filter(n=>eventMatches(n,e));return r.length===0?PASS:fail(`expected no matching ${e.type} events, found ${r.length}`)}}}function eventOrder(e){for(let t of e)validateCount(t.count);return{name:`eventOrder(${e.map(e=>e.type).join(` → `)})`,evaluate(n){let r=-1;for(let i of e){let e=n.events.flatMap((e,n)=>eventMatches(e,i)?[n]:[]);if(!matchesCount(i.count,e.length))return fail(`expected ${describeCount(i.count)} matching ${i.type} event(s), found ${e.length}`);let a=e[0];if(a===void 0||a<=r)return fail(`event group ${i.type} did not occur after the previous group`);r=e[e.length-1]??r}return PASS}}}function outputEquals(t){return{name:`outputEquals`,evaluate(n){return deepEquals(n.output,t)?PASS:fail(`output ${truncate(JSON.stringify(n.output))} does not equal expected ${truncate(JSON.stringify(t))}`)}}}function outputMatches(e){return{name:`outputMatches`,async evaluate(t){let n=await e[`~standard`].validate(t.output);return!(`issues`in n)||n.issues===void 0?PASS:fail(`output failed schema validation: ${n.issues.map(e=>e.message).join(`; `)}`)}}}function joinCompletedMessages(e){let t=[];for(let n of e)n.type===`message.completed`&&n.data.message!==null&&t.push(n.data.message);return t.join(`
`)}function failureDetail(e,t){return t===void 0?e:`${e} (code: ${t})`}function formatFailedActionResult(e){let{result:t,status:n}=e.data;return[actionResultLabel(t),`callId=${t.callId}`,`status=${n}`,t.isError===!0?`isError=true`:void 0,e.data.error?.message===void 0?void 0:`error=${e.data.error.message}`,`output=${truncate(formatUnknown(t.output))}`].filter(e=>e!==void 0).join(` `)}function actionResultLabel(e){switch(e.kind){case`tool-result`:return`tool-result:${e.toolName}`;case`subagent-result`:return`subagent-result:${e.subagentName}`;case`load-skill-result`:return e.name===void 0?`load-skill-result`:`load-skill-result:${e.name}`}}function runFailure(e){if(e.status===`failed`)return fail(failureDetail(`run failed`,e.derived.failureCode));let t=e.events.find(e=>e.type===`turn.failed`||e.type===`step.failed`);return t===void 0?void 0:fail(`${t.type} (${t.data.code}): ${t.data.message}`)}function truncate(e,t=200){return e===void 0?`undefined`:e.length<=t?e:`${e.slice(0,t)}…`}function formatUnknown(e){if(e!==void 0){if(typeof e==`string`)return e;try{return JSON.stringify(e)}catch{return String(e)}}}function matchesCount(e,t){return e===void 0?t>0:typeof e==`function`?e(t):t===e}function describeCount(e){return e===void 0?`at least one`:typeof e==`function`?`a count satisfying the predicate`:`exactly ${e}`}function validateCount(e){if(!(e===void 0||typeof e==`function`)&&(typeof e!=`number`||!Number.isInteger(e)||e<0))throw TypeError(`Assertion count must be a non-negative integer or predicate; received ${String(e)}.`)}function missingOrderedName(e,t){let n=0;for(let r of t)if(r===e[n]&&(n+=1),n===e.length)return;let r=e[n];return r===void 0?void 0:{cursor:n,name:r}}function requestedTools(e){let t=[],n=new Set,append=(e,r)=>{n.has(e)||(n.add(e),t.push({callId:e,name:r}))};for(let t of e)if(t.type===`actions.requested`)for(let e of t.data.actions)e.kind===`tool-call`&&append(e.callId,e.toolName);else if(t.type===`input.requested`)for(let e of t.data.requests){let{action:t}=e;t.kind===`tool-call`&&append(t.callId,t.toolName)}return t}export{calledSubagent,calledTool,eventOrder,eventsSatisfy,loadedSkill,maxToolCalls,messageIncludes,noFailedActions,notCalledTool,notEvent,outputEquals,outputMatches,parked,succeeded,toolOrder,typedEvent,usedNoTools};