UNPKG

eve

Version:

Filesystem-first framework for durable backend AI agents that run anywhere.

github.com/vercel/eve

vercel/eve

3 lines • 7.86 kB
1
2
3
import{createRequire}from"node:module";import{parseJsonValue}from"#shared/json.js";import{resolveLocalGitMetadata}from"#evals/runner/resolve-git-metadata.js";function Datadog(e={}){return new DatadogReporter(e)}var DatadogReporter=class{#e;#t=new Map;#n;#r;#i;#a;constructor(e){this.#e=e}async onRunStart(e,t){this.#n=void 0,this.#r=void 0,this.#i=void 0,this.#a=void 0,this.#t.clear();for(let t of e)this.#t.set(t.id,t);let n=await resolveDatadogClient(this.#e,e),r=resolveLocalGitMetadata(process.cwd()),i=this.#e.experimentName??defaultExperimentName(),a=resolveExperimentMetadata(e,t);r.sha&&(a.eveGitCommit=r.sha,a.eveGitBranch=r.branch),Object.assign(a,this.#e.metadata);let o={name:i,projectName:resolveProjectName(this.#e,e),description:this.#e.description,dataset:{name:this.#e.datasetName??`${i} dataset`},tags:resolveExperimentTags(this.#e,t),metadata:toDatadogJsonRecord(a),config:toDatadogJsonRecord(this.#e.config)};if(this.#e.recordInputs){if(!n.createDataset)throw Error(`The installed 'dd-trace' package does not expose tracer.llmobs.experiments.createDataset().`);this.#n=n,this.#r=o;return}this.#i=await n.startExperiment(o)}async onEvalComplete(e){!this.#e.recordInputs&&this.#i&&await this.#s(e)}async onRunComplete(e){try{if(this.#e.recordInputs&&await this.#o(e.results),!this.#i)return;let t=e.failed>0||e.errored>0;await this.#i.close({status:t?`failed`:`completed`,error:t?`${e.failed} failed, ${e.errored} errored`:void 0});let n=this.#e.log??console.log;this.#a&&n(`Datadog dataset URL: ${this.#a}\n`);let r=this.#i.url();r&&n(`Datadog experiment URL: ${r}\n`)}finally{this.#n=void 0,this.#r=void 0,this.#i=void 0,this.#a=void 0}}async#o(e){let t=this.#n,n=this.#r;if(!t?.createDataset||!n)return;let r=n.dataset?.name??`${n.name} dataset`,i=e.map(e=>{let t=this.#t.get(e.id),n={inputData:toDatadogJsonValue(resolveInput(e,t)),metadata:{eveEvalId:e.id}};if(this.#e.recordExpectedOutputs){let e=toOptionalDatadogJsonValue(resolveExpectedOutput(t));e!==void 0&&(n.expectedOutput=e)}return n}),a=t.createDataset(r,{projectName:n.projectName,description:this.#e.description??`Eve eval inputs for experiment '${n.name}'.`,records:i});await a.push();let o=a.id();if(!o)throw Error(`Datadog dataset '${r}' has no id after push().`);let s=a.records();if(s.length!==e.length)throw Error(`Datadog dataset '${r}' has ${s.length} records for ${e.length} eval results.`);let c={id:o,name:a.name()},l=a.version();l!==null&&(c.version=l),this.#a=a.url()??void 0,this.#i=await t.startExperiment({...n,dataset:c});for(let[t,n]of e.entries()){let e=s[t]?.id;if(!e)throw Error(`Datadog dataset record ${t+1} has no id after push().`);await this.#s(n,e)}}async#s(e,t){if(!this.#i)return;let n=this.#t.get(e.id),r={name:e.id,metadata:toDatadogJsonRecord(resolveResultMetadata(e,n,this.#e.recordAssertionDetails===!0)),tags:resolveResultTags(e,n),startedAt:e.startedAt,completedAt:e.completedAt,durationMs:elapsedMs(e.startedAt,e.completedAt)};this.#e.recordInputs&&(r.input=toOptionalDatadogJsonValue(resolveInput(e,n))),this.#e.recordOutputs&&(r.output=toOptionalDatadogJsonValue(e.result.output)),this.#e.recordExpectedOutputs&&(r.expectedOutput=toOptionalDatadogJsonValue(resolveExpectedOutput(n))),this.#e.recordErrors&&e.error!==void 0&&(r.error=e.error),t!==void 0&&(r.datasetRecordId=t);let i=await this.#i.submitSpan(r);await this.#i.submitEvaluationMetrics(i,resolveEvaluationMetrics(e,this.#e.recordAssertionDetails===!0))}};const DD_TRACE_PACKAGE=`dd-trace`,EXPECTED_OUTPUT_METADATA_KEYS=new Set([`expectedOutput`,`expected`,`expected_output`]),BUILT_IN_METRIC_LABELS=[`eve_tool_call_count`,`eve_subagent_call_count`,`eve_message_count`,`eve_reasoning_block_count`];async function resolveDatadogClient(e,t){if(e.client)return e.client;let n=await loadDatadogSdk(),r=resolveProjectName(e,t),i=n.init({service:e.service??process.env.DD_SERVICE??r,env:e.env??process.env.DD_ENV,site:e.site??process.env.DD_SITE,llmobs:{projectName:r,mlApp:e.mlApp??r,agentlessEnabled:!0}}).llmobs?.experiments;if(!i?.startExperiment)throw Error([`The installed 'dd-trace' package does not expose tracer.llmobs.experiments.startExperiment().`,`Update to a release compatible with dd-trace@6.13.0.`].join(` `));return i}async function loadDatadogSdk(){try{return createRequire(`${process.cwd()}/package.json`)(DD_TRACE_PACKAGE)}catch{try{let e=await import(DD_TRACE_PACKAGE);return e.default??e}catch{throw Error([`The 'dd-trace' package is required for Datadog reporting but was not found.`,``,`Install the tested release with:`,` npm install dd-trace@6.13.0`].join(` `))}}}function resolveProjectName(e,t){return e.projectName??process.env.DD_LLMOBS_PROJECT_NAME??e.mlApp??process.env.DD_LLMOBS_ML_APP??process.env.DD_SERVICE??t[0]?.id??`eve evals`}function defaultExperimentName(){return`eve evals ${new Date().toISOString()}`}function resolveExperimentTags(e,t){return{source:`eve`,target_kind:t.kind,...e.tags}}function resolveExperimentMetadata(e,t){let n={eveEvalIds:e.map(e=>e.id),eveTargetKind:t.kind,eveTimestamp:new Date().toISOString()},r=resolveTargetOrigin(t.url);return r!==void 0&&(n.eveTargetOrigin=r),n}function resolveTargetOrigin(e){try{let t=new URL(e).origin;return t===`null`?void 0:t}catch{return}}function resolveResultTags(e,t){let n={eval_id:e.id,eval_verdict:e.verdict,eval_status:e.result.status};return t?.tags?.length&&(n.eval_tags=t.tags.join(`,`)),n}function resolveInput(e,t){for(let t of e.result.events)if(t.type===`message.received`)return t.data.message;return t?.description??``}function resolveExpectedOutput(e){if(e?.metadata)return e.metadata.expectedOutput??e.metadata.expected??e.metadata.expected_output}function resolveResultMetadata(e,t,n){let r={};for(let[e,n]of Object.entries(t?.metadata??{}))EXPECTED_OUTPUT_METADATA_KEYS.has(e)||(r[e]=n);if(Object.assign(r,{eveSessionId:e.result.sessionId,eveStatus:e.result.status,eveVerdict:e.verdict,eveSkipReason:e.skipReason,eveToolCalls:e.result.derived.toolCalls.map(e=>e.name),eveSubagentCalls:e.result.derived.subagentCalls.map(e=>e.name),eveParked:e.result.derived.parked}),n){let t=e.assertions.filter(e=>!e.passed).map(e=>({name:e.name,message:e.message}));t.length>0&&(r.eveFailedAssertions=t)}return e.result.derived.failureCode&&(r.eveFailureCode=e.result.derived.failureCode),r}function resolveEvaluationMetrics(e,t){let n=[],r=new Set(BUILT_IN_METRIC_LABELS);for(let[i,a]of e.assertions.entries()){let e=a.severity===`gate`?`gate_${a.name}`:a.name,o={assertion_index:String(i+1),assertion_severity:a.severity,assertion_passed:String(a.passed)};t&&(o.assertion_name=a.name,o.assertion_label=e),n.push({label:reserveDatadogMetricLabel(toDatadogMetricLabel(e),r),value:a.score,tags:o})}return n.push({label:`eve_tool_call_count`,value:e.result.derived.toolCallCount},{label:`eve_subagent_call_count`,value:e.result.derived.subagentCallCount},{label:`eve_message_count`,value:e.result.derived.messageCount},{label:`eve_reasoning_block_count`,value:e.result.derived.reasoningBlockCount}),n}function toDatadogMetricLabel(e){return e.replace(/[^a-zA-Z0-9_-]+/g,`_`).replace(/^_+|_+$/g,``)||`metric`}function reserveDatadogMetricLabel(e,t){let n=e,r=2;for(;t.has(n);)n=`${e}_${r}`,r+=1;return t.add(n),n}function elapsedMs(e,t){let n=Date.parse(e),r=Date.parse(t);if(Number.isFinite(n)&&Number.isFinite(r))return Math.max(0,r-n)}function toOptionalDatadogJsonValue(e){return e===void 0?void 0:toDatadogJsonValue(e)}function toDatadogJsonRecord(e){if(e===void 0)return;let t={};for(let[n,r]of Object.entries(e))r!==void 0&&(t[n]=toDatadogJsonValue(r));return t}function toDatadogJsonValue(e){return cloneDatadogJsonValue(parseJsonValue(e))}function cloneDatadogJsonValue(e){if(Array.isArray(e))return e.map(e=>cloneDatadogJsonValue(e));if(typeof e==`object`&&e){let t={};for(let[n,r]of Object.entries(e))t[n]=cloneDatadogJsonValue(r);return t}return e}export{Datadog};