eve/dist/src/evals/session.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{basename,extname}from"node:path";import{readFile}from"node:fs/promises";import{deriveResultStatus,extractCompletedMessage,extractInputRequests}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{extractCompletedResult}from"#client/output-schema.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=[];#r;#i=[];constructor(e){this.#e=e.session,this.#t=e.signal}get events(){return this.#n}get lastTurn(){return this.#r}get pendingInputRequests(){return this.#i}get sessionId(){return this.#e.state.sessionId??this.#r?.sessionId}get state(){return this.#e.state}expectInputRequests(e){if(this.#i.length===0)throw Error(`Expected pending input requests, but the last turn did not park.`);let t=this.#i.filter(t=>inputRequestMatches(t,e));if(t.length===0)throw Error(`No pending input requests matched ${formatInputRequestFilter(e)}.`);return t}async respond(...e){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await this.send({inputResponses:e})}async respondAll(e){let t=this.expectInputRequests();for(let n of t)assertRequestHasOption(n,e);return await this.respond(...t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e){let t=await(await this.#e.send(attachSignal(e,this.#t))).result();return this.#a({data:t.data,events:t.events,inputRequests:t.inputRequests,message:t.message,sessionId:t.sessionId,status:t.status})}async sendFile(t,r,i){let a=createTextWithFileContent({bytes:await readFile(r),filename:basename(r),mediaType:i??inferMediaType(r),text:t});return await this.send({message:a})}async readTurn(e){let t=this.sessionId,n=[],o=!1;for await(let t of this.#e.stream({signal:this.#t,startIndex:e?.startIndex}))if(n.push(t),isCurrentTurnBoundaryEvent(t)){o=!0;break}if(!o)throw Error(`Stream for session "${this.sessionId??`(unknown)`}" closed before a turn boundary.`);return this.#a({data:extractCompletedResult(n),events:n,inputRequests:extractInputRequests(n),message:extractCompletedMessage(n),sessionId:t,status:deriveResultStatus(n)})}snapshot(e){let t=this.sessionId;return{derived:deriveRunFacts(this.#n,{sessionId:t}),events:[...this.#n],primary:e,sessionId:t,state:this.#e.state}}#a(e){this.#n.push(...e.events),this.#i=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({data:e.data,events:e.events,inputRequests:e.inputRequests,message:e.message,sessionId:e.sessionId??this.sessionId??``,status:e.status,toolCalls:t.toolCalls});return this.#r=n,n}},EvalTurn=class{data;events;inputRequests;message;sessionId;status;toolCalls;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.status=e.status,this.toolCalls=e.toolCalls}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}},EvalSessionManager=class{#e;#t;#n=[];#r;constructor(e){this.#e=e.client,this.#t=e.signal}get primary(){return this.#r??=this.#i(),this.#r}newSession(){return this.#i()}async attachSession(e,t){let n=new EvalSessionDriver({session:this.#e.session({sessionId:e,streamIndex:t?.startIndex??0}),signal:this.#t});return this.#n.push(n),await n.readTurn(t),n}snapshots(){return this.#n.map(e=>e.snapshot(e===this.#r))}lastTurnSession(){return this.#r?.lastTurn===void 0?this.#n.find(e=>e.lastTurn!==void 0):this.#r}#i(){let e=new EvalSessionDriver({session:this.#e.session(),signal:this.#t});return this.#n.push(e),e}};function attachSignal(e,t){if(t===void 0)return e;if(typeof e==`string`)return{message:e,signal:t};let n=e;return n.signal===void 0?{...n,signal:t}:n}function inputRequestMatches(e,t){return t===void 0?!0:t.display!==void 0&&e.display!==t.display?!1:t.toolName===void 0?!0:e.action.kind===`tool-call`&&e.action.toolName===t.toolName}function formatInputRequestFilter(e){return e===void 0?`{}`:JSON.stringify(e)}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}function inferMediaType(e){switch(extname(e).toLowerCase()){case`.gif`:return`image/gif`;case`.jpg`:case`.jpeg`:return`image/jpeg`;case`.png`:return`image/png`;case`.webp`:return`image/webp`;default:return`application/octet-stream`}}export{EvalSessionDriver,EvalSessionManager,EveEvalTurnFailedError};