eve
Version:
Filesystem-first framework for durable backend AI agents that run anywhere.
1 lines • 2.71 kB
JavaScript
import{join,relative}from"node:path";import{access,readdir}from"node:fs/promises";import{loadAuthoredModuleNamespace}from"#internal/authored-module-loader.js";const EVAL_FILE_SUFFIX=`.eval.ts`,EVAL_CONFIG_FILE=`evals.config.ts`;async function discoverEvalFiles(n){let r=join(n,`evals`),i=[];try{await collectEvalFiles(r,i)}catch(e){if(isNoEntryError(e))return[];throw e}return i.sort((e,n)=>relative(r,e).localeCompare(relative(r,n))),i}function deriveEvalId(n,r){let i=relative(join(n,`evals`),r).split(/[\\/]/u).join(`/`);return i.endsWith(EVAL_FILE_SUFFIX)?i.slice(0,-8):i}function matchesEvalFilter(e,t){return t.length===0?!0:t.some(t=>e===t||e.startsWith(`${t}/`))}async function importEvalFile(e,t){let n=(await loadAuthoredModuleNamespace(t)).default,r=deriveEvalId(e,t);if(isEveEvalDefinition(n))return[{...n,id:r}];if(Array.isArray(n))return n.map((e,n)=>{if(!isEveEvalDefinition(e))throw Error(`Eval file "${t}" exports an array whose entry at index ${n} is not a valid EveEval. Use defineEval() to create every entry.`);return{...e,id:`${r}/${String(n).padStart(4,`0`)}`}});throw Error(`Eval file "${t}" does not export a valid EveEval (or array of EveEvals) as its default export. Use defineEval() to create the eval.`)}async function discoverAndImportEvals(e,t){let n=await discoverEvalFiles(e);if(n.length===0)return[];let r=t??[],i=[],a=new Map;for(let t of n)for(let n of await importEvalFile(e,t)){let e=a.get(n.id);if(e!==void 0)throw Error(`Duplicate eval id "${n.id}" derived from both "${e}" and "${t}".`);a.set(n.id,t),matchesEvalFilter(n.id,r)&&i.push(n)}return i}async function discoverEvalConfig(t){let r=join(t,`evals`,EVAL_CONFIG_FILE);try{await access(r)}catch(e){throw isNoEntryError(e)?Error(`Missing required eval config at evals/${EVAL_CONFIG_FILE}. Create it with defineEvalConfig({}) (optionally \`{ judge: { model } }\` to set the default judge model for \`t.judge.*\` assertions).`):e}let a=(await loadAuthoredModuleNamespace(r)).default;if(!isEveEvalConfig(a))throw Error(`Eval config "evals/${EVAL_CONFIG_FILE}" must default-export a defineEvalConfig() value.`);return a}function isEveEvalConfig(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEvalConfig`}function isEveEvalDefinition(e){return typeof e==`object`&&!!e&&`_tag`in e&&e._tag===`EveEval`}async function collectEvalFiles(t,n){let i=await readdir(t,{withFileTypes:!0});for(let r of i){let i=join(t,r.name);r.isDirectory()?await collectEvalFiles(i,n):r.isFile()&&r.name.endsWith(EVAL_FILE_SUFFIX)&&n.push(i)}}function isNoEntryError(e){return typeof e==`object`&&!!e&&`code`in e&&e.code===`ENOENT`}export{discoverAndImportEvals,discoverEvalConfig,discoverEvalFiles,importEvalFile,matchesEvalFilter};