@mesh-tech/mesh-cli
Version:
CLI for Mesh platform development utilities
252 lines (251 loc) • 8.97 kB
JavaScript
import { parseArgs } from "node:util";
const { values: args } = parseArgs({
options: {
prompt: { type: "string", short: "p" },
"prompt-file": { type: "string", short: "f" },
api: { type: "string", default: "http://localhost:8787" },
conversation: { type: "string", short: "c" },
timeout: { type: "string", default: "300" },
verbose: { type: "boolean", short: "v", default: false },
},
strict: true,
});
const apiUrl = (args.api ?? "http://localhost:8787").replace(/\/+$/, "");
const timeoutMs = parseInt(args.timeout ?? "300", 10) * 1000;
let prompt;
if (args.prompt) {
prompt = args.prompt;
}
else if (args["prompt-file"]) {
const fs = await import("node:fs");
prompt = fs.readFileSync(args["prompt-file"], "utf-8").trim();
}
else {
console.error("Usage: agent-eval.mts --prompt '...' or --prompt-file <path>");
process.exit(1);
}
async function* readSSE(response) {
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
while (true) {
const { done, value } = await reader.read();
if (done)
break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split("\n");
buffer = lines.pop() ?? "";
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed || !trimmed.startsWith("data: "))
continue;
const data = trimmed.slice(6);
if (data === "[DONE]")
return;
try {
yield JSON.parse(data);
}
catch {
}
}
}
}
const log = (msg) => process.stderr.write(msg + "\n");
const vlog = (msg) => { if (args.verbose)
log(` ${msg}`); };
log(`\n--- Agent Eval ---`);
log(`API: ${apiUrl}`);
log(`Prompt: ${prompt.slice(0, 100)}${prompt.length > 100 ? "..." : ""}`);
log("");
const startTime = Date.now();
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
const trace = {
conversationId: args.conversation ?? null,
prompt,
apiUrl,
startedAt: new Date().toISOString(),
durationMs: 0,
toolCalls: [],
textSegments: [],
artifacts: [],
title: null,
finishReason: null,
usage: null,
errors: [],
warnings: [],
};
const pendingTools = new Map();
let currentText = "";
try {
const response = await fetch(`${apiUrl}/v1/conversations/messages`, {
method: "POST",
headers: {
"Content-Type": "application/json",
"X-Forwarded-User": "eval-runner",
"X-Forwarded-Email": "eval@localhost",
},
body: JSON.stringify({
conversationId: trace.conversationId,
messages: [{ role: "user", content: prompt }],
stream: true,
}),
signal: controller.signal,
});
if (!response.ok) {
const body = await response.text();
log(`ERROR: API returned ${response.status}: ${body}`);
process.exit(1);
}
for await (const event of readSSE(response)) {
const type = event.type;
switch (type) {
case "conversation-started":
trace.conversationId = event.conversationId;
log(`Conversation: ${trace.conversationId}`);
break;
case "title-updated":
trace.title = event.title;
vlog(`Title: ${trace.title}`);
break;
case "text-start":
currentText = "";
break;
case "text-delta":
currentText += event.delta;
break;
case "text-end":
if (currentText) {
trace.textSegments.push(currentText);
vlog(`Text: ${currentText.slice(0, 80)}${currentText.length > 80 ? "..." : ""}`);
}
currentText = "";
break;
case "tool-call-start": {
const tc = {
toolId: event.toolId,
toolName: event.toolName,
args: event.args,
startedAt: Date.now(),
};
pendingTools.set(tc.toolId, tc);
trace.toolCalls.push(tc);
log(` -> ${tc.toolName}${args.verbose ? ` (${JSON.stringify(tc.args).slice(0, 120)})` : ""}`);
break;
}
case "tool-call-result": {
const tc = pendingTools.get(event.toolId);
if (tc) {
tc.result = event.result;
tc.durationMs = Date.now() - tc.startedAt;
pendingTools.delete(tc.toolId);
const r = event.result;
if (r && r.success === false) {
const errMsg = r.error ?? "success: false";
trace.errors.push(`${tc.toolName}: ${errMsg}`);
log(` <- ${tc.toolName} FAILED: ${errMsg.slice(0, 100)}`);
}
else {
vlog(` <- ${tc.toolName} OK (${tc.durationMs}ms)`);
}
if (r && Array.isArray(r.warnings)) {
for (const w of r.warnings) {
trace.warnings.push(`${tc.toolName}: ${w}`);
}
}
if (r && r.action_required) {
trace.warnings.push(`${tc.toolName} ACTION REQUIRED: ${r.action_required}`);
}
if (r && Array.isArray(r.missing_specs)) {
for (const w of r.missing_specs) {
trace.warnings.push(`${tc.toolName} MISSING SPEC: ${w}`);
}
}
}
break;
}
case "tool-call-error": {
const tc = pendingTools.get(event.toolId);
if (tc) {
tc.error = event.error;
tc.durationMs = Date.now() - tc.startedAt;
pendingTools.delete(tc.toolId);
trace.errors.push(`${tc.toolName}: ${tc.error}`);
log(` <- ${tc.toolName} ERROR: ${tc.error.slice(0, 100)}`);
}
break;
}
case "artifact-update": {
const a = event.artifact;
const entry = { id: a.id, type: a.type, title: a.title };
const idx = trace.artifacts.findIndex((x) => x.id === entry.id);
if (idx >= 0)
trace.artifacts[idx] = entry;
else
trace.artifacts.push(entry);
vlog(`Artifact: ${entry.id} (${entry.type}): ${entry.title}`);
break;
}
case "error":
trace.errors.push(event.error);
log(` ERROR: ${event.error}`);
break;
case "finish":
trace.finishReason = event.finishReason;
trace.usage = event.usage ?? null;
break;
}
}
}
catch (err) {
if (err.name === "AbortError") {
trace.errors.push(`Timeout after ${timeoutMs / 1000}s`);
log(`\nTIMEOUT after ${timeoutMs / 1000}s`);
}
else {
trace.errors.push(err.message);
log(`\nERROR: ${err.message}`);
}
}
finally {
clearTimeout(timer);
}
trace.durationMs = Date.now() - startTime;
log("\n--- Summary ---");
log(`Duration: ${(trace.durationMs / 1000).toFixed(1)}s`);
log(`Tool calls: ${trace.toolCalls.length}`);
log(`Text segments: ${trace.textSegments.length}`);
log(`Artifacts: ${trace.artifacts.length}`);
log(`Errors: ${trace.errors.length}`);
log(`Warnings: ${trace.warnings.length}`);
log(`Finish: ${trace.finishReason ?? "unknown"}`);
if (trace.usage) {
log(`Tokens: ${trace.usage.inputTokens} in / ${trace.usage.outputTokens} out`);
}
if (trace.errors.length > 0) {
log("\nErrors:");
for (const e of trace.errors)
log(` - ${e}`);
}
if (trace.warnings.length > 0) {
log("\nWarnings:");
for (const w of trace.warnings)
log(` - ${w}`);
}
const toolCounts = new Map();
for (const tc of trace.toolCalls) {
const entry = toolCounts.get(tc.toolName) ?? { count: 0, failed: 0 };
entry.count++;
if (tc.error || tc.result?.success === false)
entry.failed++;
toolCounts.set(tc.toolName, entry);
}
if (toolCounts.size > 0) {
log("\nTool breakdown:");
for (const [name, { count, failed }] of toolCounts) {
log(` ${name}: ${count} call${count > 1 ? "s" : ""}${failed > 0 ? ` (${failed} failed)` : ""}`);
}
}
log("");
console.log(JSON.stringify(trace, null, 2));