eve/dist/src/evals/session.js

Filesystem-first framework for durable backend AI agents that run anywhere.

import{readFile}from"node:fs/promises";import{basename}from"node:path";import{isCurrentTurnBoundaryEvent,isTurnFailureEvent}from"#protocol/message.js";import{summarizeTurnEvents}from"#client/session-utils.js";import{createTextWithFileContent}from"#client/file-parts.js";import{extractCompletedResult}from"#client/output-schema.js";import{inputRequestMatches,matchesValue,toolCallMatches}from"#evals/match.js";import"#evals/assertions/collector.js";import{createOutputAssertions,createScopedAssertions}from"#evals/assertions/scoped.js";import{EvalRequirementFailed}from"#evals/control-flow.js";import{deriveRunFacts}from"#evals/runner/derive-run-facts.js";import{formatEvalTranscript,inferMediaType}from"#evals/session-content.js";var EveEvalTurnFailedError=class extends Error{event;turn;constructor(e){let t=e.events.find(isTurnFailureEvent),n=t===void 0?`turn ended with status "${e.status}"`:`${t.type}: ${t.data.code} ${t.data.message}`.trim();super(`Eval turn failed: ${n}`),this.name=`EveEvalTurnFailedError`,this.event=t,this.turn=e}},EvalSessionDriver=class{#e;#t;#n=``;#r;#i;#a=[];#o;#s;#c=[];#l=new Set;#u;#d=!1;#f=[];constructor(e){this.#i=e.collector,this.#s=e.onSessionStart,this.#o=e.primary,this.#e=e.session,this.#t=e.onTurn,this.#r=e.signal,Object.assign(this,createScopedAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}),createOutputAssertions(this.#i,{timing:`snapshot`,select:()=>this.#_()}))}get events(){return this.#a}get transcript(){return formatEvalTranscript(this.#a)}get lastInput(){return this.#n}get lastTurn(){return this.#u}get pendingInputRequests(){return this.#f}get sessionId(){return this.#e.state.sessionId}get state(){return this.#e.state}async cancel(){return await this.#e.cancel()}async cleanup(e){await this.#e.reset({reason:`Eval timed out`,signal:e})}requireInputRequest(e={}){this.#f.length===0&&this.#v(`requireInputRequest`,`expected one pending input request, but the last turn did not park`);let t=this.#f.filter(t=>inputRequestMatches(t,e));return(this.#f.length!==1||t.length!==1)&&this.#v(`requireInputRequest`,`expected exactly one pending input request matching ${formatInputRequestFilter(e)}, found ${t.length} match(es) across ${this.#f.length} pending request(s)`),this.#i.recordOutcome({name:`requireInputRequest`,outcome:{score:1}}),t[0]}async respond(e,t={}){if(e.length===0)throw Error(`respond() requires at least one input response.`);return await(await this.#p({...t,inputResponses:e})).result()}async startRespond(e,t={}){if(e.length===0)throw Error(`startRespond() requires input responses.`);return await this.#p({...t,inputResponses:e})}async respondAll(e){let t=this.#f;if(t.length===0)throw Error(`respondAll() requires at least one pending input request.`);for(let n of t)assertRequestHasOption(n,e);return await this.respond(t.map(t=>({optionId:e,requestId:t.requestId})))}async send(e,t={}){return await(await this.#p({turnPolicy:`queue`,...t,message:e})).result()}async start(e,t={}){return await this.#p({turnPolicy:`queue`,...t,message:e})}async#p(e){let{inputResponses:t,message:n,...r}=attachSignal(e,this.#r),i=t===void 0?await this.#e.send(n,r):await this.#e.respond(t,r);return this.consume(i,n)}consume(e,t){let n=this.sessionId;return new EvalLiveTurn({events:e,observe:e=>this.#m(n,e),record:e=>(t!==void 0&&(this.#n=typeof t==`string`?t:t.filter(e=>e.type===`text`).map(e=>e.text).join(`
`)),this.#g(n,e)),session:this,sessionId:n})}async sendFile(e,t,n){let r=await readFile(t),i=createTextWithFileContent({bytes:r,filename:basename(t),mediaType:n??inferMediaType(t),text:e});return await this.send(i)}async readTurn(e){return await this.watchTurn(e).result()}watchTurn(e){return this.consume(this.#e.stream({signal:this.#r,startIndex:e?.startIndex}))}snapshot(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],primary:this.#o,sessionId:e,state:this.#e.state,traceContexts:[...this.#c]}}#m(e,t){if(t.type!==`session.started`&&t.type!==`turn.started`)return;let n=t.data.trace;if(n===void 0)return;let r=`${n.traceId}:${n.spanId}`;this.#l.has(r)||(this.#l.add(r),this.#c.push(n),!this.#d&&(this.#d=!0,this.#s?.({primary:this.#o,sessionId:e,startedAt:t.meta.at,traceContext:n})))}#h(e){this.#a.push(...e.events),this.#f=e.status===`waiting`?e.inputRequests:[];let t=deriveRunFacts(e.events,{sessionId:e.sessionId}),n=new EvalTurn({collector:this.#i,data:e.data,derived:t,events:e.events,inputRequests:e.inputRequests,message:e.message,session:this,sessionId:e.sessionId,status:e.status,toolCalls:t.toolCalls});return this.#u=n,this.#t(this),n}#g(e,t){let n=summarizeTurnEvents(t);return this.#h({data:extractCompletedResult(t),events:t,inputRequests:n.inputRequests,message:n.message,sessionId:e,status:n.status})}#_(){let e=this.sessionId;return{derived:deriveRunFacts(this.#a,{sessionId:e}),events:[...this.#a],output:outputOf(this.#u),status:this.#u?.status??`completed`}}#v(e,t){throw this.#i.recordOutcome({name:e,outcome:{score:0,message:t}}),new EvalRequirementFailed}},EvalLiveTurn=class{session;sessionId;#e;#t=[];#n=new Set;#r;constructor(e){this.session=e.session,this.sessionId=e.sessionId,this.#e=this.#i(e.events,e.observe,e.record),this.#e.catch(()=>{})}get events(){return this.#t}async cancel(){return await this.session.cancel()}async result(){return await this.#e}async waitForEvent(e,t){let matches=n=>n.type===e&&(t?.data===void 0||matchesValue(t.data,`data`in n?n.data:void 0)),n=this.#t.find(matches);if(n!==void 0)return n;if(this.#r!==void 0)throw this.#r;return await new Promise((e,t)=>{let n={matches,reject:t,resolve:t=>e(t)};this.#n.add(n)})}async#i(e,t,n){try{let r=!1;for await(let n of e)if(this.#t.push(n),t(n),this.#a(n),isTurnFailureEvent(n)&&this.#o(Error(`Session ${this.sessionId} failed before the expected event (${n.type}).`)),isCurrentTurnBoundaryEvent(n)){r=!0,this.#o(Error(`Session ${this.sessionId} reached ${n.type} before the expected event.`));break}if(!r)throw Error(`Stream for session "${this.sessionId}" closed before a turn boundary.`);return n(this.#t)}catch(e){let t=e instanceof Error?e:Error(String(e));throw this.#o(t),e}}#a(e){for(let t of this.#n)t.matches(e)&&(this.#n.delete(t),t.resolve(e))}#o(e){if(this.#r===void 0){this.#r=e;for(let t of this.#n)t.reject(e);this.#n.clear()}}},EvalTurn=class{data;events;inputRequests;message;session;sessionId;status;toolCalls;#e;#t;constructor(e){this.data=e.data,this.events=e.events,this.inputRequests=e.inputRequests,this.message=e.message,this.sessionId=e.sessionId,this.session=e.session,this.status=e.status,this.toolCalls=e.toolCalls,this.#e=e.collector,this.#t=e.derived,Object.assign(this,createScopedAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}),createOutputAssertions(e.collector,{timing:`snapshot`,select:()=>this.#n()}))}expectOk(){if(this.status!==`failed`)return this;throw new EveEvalTurnFailedError(this)}requireToolCall(e,t={}){let n=this.toolCalls.filter(n=>n.name===e&&toolCallMatches(n,t));return n.length!==1&&inputRequirementFailed(this.#e,`requireToolCall`,`expected exactly one matching "${e}" tool call in this turn, found ${n.length}; observed [${this.toolCalls.map(e=>e.name).join(`, `)}]`),this.#e.recordOutcome({name:`requireToolCall`,outcome:{score:1}}),n[0]}#n(){return{derived:this.#t,events:this.events,output:outputOf(this),status:this.status}}};function attachSignal(e,t){return t===void 0?e:e.signal===void 0?{...e,signal:t}:e}function formatInputRequestFilter(e){return JSON.stringify(e)}function inputRequirementFailed(e,t,n){throw e.recordOutcome({name:t,outcome:{score:0,message:n}}),new EvalRequirementFailed}function outputOf(e){return e===void 0?null:e.data===void 0?e.message??null:e.data}function assertRequestHasOption(e,t){if(e.options===void 0||e.options.length===0)throw Error(`Input request "${e.requestId}" has no selectable options.`);if(!e.options.some(e=>e.id===t))throw Error(`Input request "${e.requestId}" does not offer option "${t}".`)}export{EvalSessionDriver,EveEvalTurnFailedError};