eve
Version:
Filesystem-first framework for durable backend AI agents that run anywhere.
1 lines • 3.21 kB
JavaScript
import{access,readdir}from"node:fs/promises";import{join,relative}from"node:path";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";const EVAL_FILE_SUFFIX=`.eval.ts`,EVAL_CONFIG_FILE=`evals.config.ts`;async function discoverEvalFiles(e){let t=join(e,`evals`),n=[];try{await collectEvalFiles(t,n)}catch(e){if(isNoEntryError(e))return[];throw e}return n.sort((e,n)=>relative(t,e).localeCompare(relative(t,n))),n}async function findMisplacedEvalDirs(e){let t=join(e,`agent`),n=new Set;try{await collectMisplacedEvalDirs(t,e,n)}catch(e){if(isNoEntryError(e))return[];throw e}return[...n].sort((e,t)=>e.localeCompare(t))}function deriveEvalId(e,t){let n=join(e,`evals`),r=relative(n,t).split(/[\\/]/u).join(`/`);return r.endsWith(EVAL_FILE_SUFFIX)?r.slice(0,-8):r}function matchesEvalFilter(e,t){return t.length===0||t.some(t=>e===t||e.startsWith(`${t}/`))}async function importEvalFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default,r=deriveEvalId(e,t);if(isEveEvalDefinition(n))return[{...n,id:r}];if(Array.isArray(n))return n.map((e,n)=>{if(!isEveEvalDefinition(e))throw Error(`Eval file "${t}" exports an array whose entry at index ${n} is not a valid EveEval. Use defineEval() to create every entry.`);return{...e,id:`${r}/${String(n).padStart(4,`0`)}`}});throw Error(`Eval file "${t}" does not export a valid EveEval (or array of EveEvals) as its default export. Use defineEval() to create the eval.`)}async function discoverAndImportEvals(e,t){let n=await discoverEvalFiles(e);if(n.length===0)return[];let r=t??[],i=[],a=new Map;for(let t of n)for(let n of await importEvalFile(e,t)){let e=a.get(n.id);if(e!==void 0)throw Error(`Duplicate eval id "${n.id}" derived from both "${e}" and "${t}".`);a.set(n.id,t),matchesEvalFilter(n.id,r)&&i.push(n)}return i}async function discoverEvalConfig(e){let t=join(e,`evals`,EVAL_CONFIG_FILE);try{await access(t)}catch(e){throw isNoEntryError(e)?Error(`Missing required eval config at evals/${EVAL_CONFIG_FILE}. Create it with defineEvalConfig({}) (optionally \`{ judge: { model } }\` to set the default judge model for \`t.judge(...)\` assertions).`):e}let n=(await loadAuthoredModuleNamespace(t)).default;if(!isEveEvalConfig(n))throw Error(`Eval config "evals/${EVAL_CONFIG_FILE}" must default-export a defineEvalConfig() value.`);return n}function isEveEvalConfig(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEvalConfig`}function isEveEvalDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEval`}async function collectEvalFiles(e,t){let n=await readdir(e,{withFileTypes:!0});for(let r of n){let n=join(e,r.name);r.isDirectory()?await collectEvalFiles(n,t):r.isFile()&&r.name.endsWith(EVAL_FILE_SUFFIX)&&t.push(n)}}async function collectMisplacedEvalDirs(e,t,n){let r=await readdir(e,{withFileTypes:!0});for(let i of r)i.isDirectory()?await collectMisplacedEvalDirs(join(e,i.name),t,n):i.isFile()&&i.name.endsWith(EVAL_FILE_SUFFIX)&&n.add(relative(t,e).split(/[\\/]/u).join(`/`))}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportEvals,discoverEvalConfig,discoverEvalFiles,findMisplacedEvalDirs,importEvalFile,matchesEvalFilter};