eve/dist/src/evals/session.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import{AssertionCollector}from"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n;#r=[];#i;#a=[];constructor(e){this.#n=e.collector,this.#e=e.session,this.#t=e.signal,Object.assign(this,createScopedAssertions(this.#n,{timing:`snapshot`,select:()=>this.#c()}),createOutputAssertions(this.#n,{timing:`snapshot`,select:()=>this.#c()}))}get events(){return this.#r}get lastTurn(){return this.#i}get pendingInputRequests(){return this.#a}get sessionId(){return this.#e.state.sessionId??this.#i?.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}requireInputRequest(e={}){this.#a.length===0&&this.#l(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#a.filter(t=>inputRequestMatches(t,e));return(this.#a.length!==1||t.length!==1)&&this.#l(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#a.length} pending request(s)`),this.#n.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.#a;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){return await(await this.start(e)).result()}async start(e){let t=await this.#e.send(attachSignal(e,this.#t));return new EvalLiveTurn({events:t,record:e=>this.#s(t.sessionId,e),session:this,sessionId:t.sessionId})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId;return await this.watchTurn(e,requireSessionId(t)).result()}watchTurn(e,t=requireSessionId(this.sessionId)){return new EvalLiveTurn({events:this.#e.stream({signal:this.#t,startIndex:e?.startIndex}),record:e=>this.#s(t,e),session:this,sessionId:t})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:t}),events:[...this.#r],primary:e,sessionId:t,state:this.#e.state}}#o(e){this.#r.push(...e.events),this.#a=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#n,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#i=n,n}#s(e,t){return this.#o({data:extractCompletedResult(t),events:t,inputRequests:extractInputRequests(t),message:extractCompletedMessage(t),sessionId:e,status:deriveResultStatus(t)})}#c(){let e=this.sessionId;return{derived:deriveRunFacts(this.#r,{sessionId:e}),events:[...this.#r],output:outputOf(this.#i),status:this.#i?.status??`completed`}}#l(e,t){throw this.#n.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t){try{let n=!1;for await(let t of e)if(this.#t.push(t),this.#a(t),isTurnFailureEvent(t)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${t.type}).`)),isCurrentTurnBoundaryEvent(t)){n=!0,this.#o(Error(`Session ${this.sessionId} reached ${t.type} before the expected event.`));break}if(!n)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return t(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}},EvalSessionManager=class{#e;#t;#n;#r=[];#i;constructor(e){this.#e=e.client,this.#n=e.collector??new AssertionCollector,this.#t=e.signal}get primary(){return this.#i??=this.#a(),this.#i}newSession(){return this.#a()}async attachSession(e,t){let n=this.#o(e,t);return await n.readTurn(t),n}watchTurn(e,t){return this.#o(e,t).watchTurn(t,e)}snapshots(){return this.#r.map(e=>e.snapshot(e===this.#i))}lastTurnSession(){return this.#i?.lastTurn===void 0?this.#r.find(e=>e.lastTurn!==void 0):this.#i}hasActivity(){return this.#r.length>0}#a(){let e=new EvalSessionDriver({collector:this.#n,session:this.#e.session(),signal:this.#t});return this.#r.push(e),e}#o(e,t){let n=new EvalSessionDriver({collector:this.#n,session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#r.push(n),n}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function requireSessionId(e){if(e===void 0)throw Error(`Eval session produced a turn without a session id.`);return e}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};