@botpress/adk-cli
Version:
Command-line interface for the Botpress Agent Development Kit (ADK)
856 lines (852 loc) • 29.9 kB
JavaScript
// @bun
import {
ensureChatIntegrationReady
} from "./chunk-rvevacmq.js";
import {
startBackendServer,
startSpanIngestServer,
stopBackendServer,
stopSpanIngestServer
} from "./chunk-d6yjgr34.js";
import"./chunk-ssjw4jwy.js";
import"./chunk-2re833ev.js";
import"./chunk-sky56mbb.js";
import {
filterEvals,
loadEvalsFromDir
} from "./chunk-xkcrb68a.js";
import"./chunk-ttvctaf1.js";
import"./chunk-wmec2w0t.js";
import"./chunk-fr1k79kd.js";
import"./chunk-tefbm840.js";
import"./chunk-bexyahs9.js";
import {
ensureJsonOnlyFormat
} from "./chunk-7sfagm12.js";
import {
getLocalEvalStore,
setServerConfig
} from "./chunk-59ayvmxs.js";
import"./chunk-68mqsf42.js";
import {
buildDevServerHeaders,
getDevServerUrl
} from "./chunk-ndxsgd72.js";
import"./chunk-r72adjnh.js";
import {
telemetry_default
} from "./chunk-kwmsaz7n.js";
import {
getAdkVersion
} from "./chunk-26vqkz52.js";
import {
resolveCommandContext
} from "./chunk-3ahwp6fe.js";
import"./chunk-9e2nksab.js";
import"./chunk-m2h26j5f.js";
import"./chunk-8gqzjqmb.js";
import {
findAgentRootOrFail
} from "./chunk-kk3h6qaj.js";
import {
createCliLogger
} from "./chunk-gzwt1qdr.js";
import"./chunk-nxy2ya5r.js";
import"./chunk-wzj4dc7n.js";
import {
AdkError,
AgentProject,
getChatClient
} from "./chunk-p0hjqn4r.js";
import"./chunk-np5wcwfv.js";
import"./chunk-dq2xpa24.js";
import"./chunk-6w0knnta.js";
import"./chunk-40x04ckt.js";
import {
runEvalSuite
} from "./chunk-t76d8fxx.js";
import"./chunk-nh2akp42.js";
import"./chunk-0fdvzjbh.js";
import"./chunk-2a5b6azq.js";
import"./chunk-vay209b5.js";
import {
Uk
} from "./chunk-3xrpxgq4.js";
import"./chunk-rfm3jr1m.js";
import"./chunk-w346ejn9.js";
import"./chunk-knvm2anf.js";
import"./chunk-65h5trb5.js";
import"./chunk-s2akeqpw.js";
import"./chunk-6771vrjp.js";
import"./chunk-g8mm42v1.js";
import"./chunk-50hzjdck.js";
import"./chunk-nn2jb0x0.js";
import"./chunk-v8xvth6j.js";
import"./chunk-kkk13rcb.js";
import"./chunk-ytpp1kam.js";
import"./chunk-na956zz3.js";
import"./chunk-f4bw8q7c.js";
import"./chunk-0v8vgrns.js";
import"./chunk-54qt5g7m.js";
import {
__require
} from "./chunk-dhs2bg35.js";
// src/commands/adk-evals.ts
import path from "path";
// src/eval/store.ts
async function saveRunResult(agentPath, report) {
const store = getLocalEvalStore(agentPath);
await store.completeRun(report.id, report);
}
async function loadRunResult(agentPath, runId) {
return getLocalEvalStore(agentPath).loadRunResult(runId);
}
async function listRunResults(agentPath, limit = 50, sinceTs) {
return getLocalEvalStore(agentPath).listRunResults(limit, sinceTs);
}
async function getLatestRun(agentPath) {
return getLocalEvalStore(agentPath).getLatestRun();
}
// src/commands/adk-evals.ts
var INDENT = " ";
function ansi(color, s) {
const codes = {
gray: "90",
red: "31",
green: "32",
yellow: "33",
blue: "34",
cyan: "36"
};
const code = codes[color];
if (!code)
return s;
return `\x1B[${code}m${s}\x1B[0m`;
}
function ansiBold(s) {
return `\x1B[1m${s}\x1B[22m`;
}
function ansiBoldColor(color, s) {
return ansiBold(ansi(color, s));
}
var icons = {
pass: ansi("green", "\u2713"),
fail: ansi("red", "\u2717"),
error: ansi("red", "\u26A0"),
run: ansi("cyan", "\u25CF"),
dot: ansi("gray", "\xB7")
};
function dim(s) {
return ansi("gray", s);
}
function duration(ms) {
if (ms < 1000)
return `${ms}ms`;
return `${(ms / 1000).toFixed(1)}s`;
}
function truncate(s, max) {
const oneLine = s.replace(/\n/g, " ").trim();
if (oneLine.length <= max)
return oneLine;
return oneLine.slice(0, max - 1) + "\u2026";
}
function assertionSummary(report) {
const turnTotal = report.turns.reduce((n, t) => n + t.assertions.length, 0);
const turnPassed = report.turns.reduce((n, t) => n + t.assertions.filter((a) => a.pass).length, 0);
const outcomeTotal = report.outcomeAssertions?.length || 0;
const outcomePassed = report.outcomeAssertions?.filter((a) => a.pass).length || 0;
const total = turnTotal + outcomeTotal;
const passed = turnPassed + outcomePassed;
return { total, passed, failed: total - passed };
}
function turnSummaryIcons(turns) {
return turns.map((t) => t.pass ? icons.pass : icons.fail).join(" ");
}
function bar(passed, total, width = 30) {
if (total === 0)
return "";
const filled = Math.round(passed / total * width);
const empty = width - filled;
const pct = Math.round(passed / total * 100);
const color = pct === 100 ? "green" : pct >= 80 ? "yellow" : "red";
return ansi(color, "\u2501".repeat(filled)) + dim("\u2501".repeat(empty)) + ` ${pct}%`;
}
function createSpinner(text) {
const spinnerLogger = createCliLogger();
const frames = ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"];
let i = 0;
let currentText = text;
const clearLine = () => spinnerLogger.stdout(`\r\x1B[2K`);
const tick = () => {
clearLine();
spinnerLogger.stdout(`\r${INDENT}${ansi("cyan", frames[i % frames.length])} ${currentText}`);
i++;
};
let interval = setInterval(tick, 80);
return {
update(next) {
currentText = next;
},
stop(result) {
if (interval)
clearInterval(interval);
interval = null;
clearLine();
spinnerLogger.stdout(`\r${INDENT}${result}
`);
},
printAbove(line) {
if (interval)
clearInterval(interval);
clearLine();
spinnerLogger.stdout(`${line}
`);
i = 0;
interval = setInterval(tick, 80);
}
};
}
function createDevServerClient(credentials) {
return new Uk({
token: credentials.token,
apiUrl: credentials.apiUrl,
workspaceId: credentials.workspaceId,
botId: credentials.devBotId,
headers: { "x-multiple-integrations": "true" }
});
}
function isLocalUrl(url) {
try {
const host = new URL(url).hostname;
return host === "localhost" || host === "127.0.0.1" || host === "::1";
} catch {
return false;
}
}
async function isDevServerRunning(serverUrl, headers) {
try {
const res = await fetch(`${serverUrl}/api/health`, {
headers,
signal: AbortSignal.timeout(2000)
});
return res.ok;
} catch {
return false;
}
}
async function getCredentialsFromDevServer(serverUrl, headers) {
const res = await fetch(`${serverUrl}/api/config`, {
headers
});
if (!res.ok) {
throw new AdkError({
code: "DEV_SERVER_DOWN",
message: `Dev server not responding at ${serverUrl}. Is "adk dev" running?`,
expected: true
});
}
const config = await res.json();
const { token, devBotId, apiUrl } = config.credentials || {};
if (!token || !devBotId) {
throw new AdkError({
code: "MISSING_CREDENTIALS",
message: "Missing credentials from dev server config.",
expected: true
});
}
return { token, devBotId, apiUrl: apiUrl || "https://api.botpress.cloud" };
}
async function ensureDevServer(serverUrl, agentRoot) {
const noop = () => {};
const headers = buildDevServerHeaders(agentRoot);
if (await isDevServerRunning(serverUrl, headers)) {
const credentials2 = await getCredentialsFromDevServer(serverUrl, headers);
return { credentials: credentials2, serverUrl, devServerHeaders: headers, cleanup: noop, lightweight: false };
}
if (!isLocalUrl(serverUrl)) {
throw new AdkError({
code: "DEV_SERVER_DOWN",
message: `Dev server not responding at ${serverUrl}. Is "adk dev" running?`,
expected: true
});
}
const context = await resolveCommandContext({
cwd: agentRoot,
target: "dev",
require: ["project", "credentials", "workspace", "bot"]
});
const project = context.project;
const devId = context.botId;
const credentials = context.credentials;
if (!credentials.token) {
throw new AdkError({
code: "AUTH_REQUIRED",
message: "No authentication token found. Run `adk login` first.",
expected: true
});
}
const backend = startBackendServer(agentRoot);
startSpanIngestServer();
setServerConfig({
port: backend.port,
agentPath: agentRoot,
project,
adkVersion: getAdkVersion(),
credentials: {
token: credentials.token,
apiUrl: credentials.apiUrl,
workspaceId: credentials.workspaceId,
devBotId: devId
}
});
const devCredentials = {
token: credentials.token,
devBotId: devId,
apiUrl: credentials.apiUrl || "https://api.botpress.cloud",
workspaceId: credentials.workspaceId
};
return {
credentials: devCredentials,
serverUrl: `http://127.0.0.1:${backend.port}`,
devServerHeaders: {},
cleanup: () => {
stopSpanIngestServer();
stopBackendServer();
},
lightweight: true
};
}
function renderEvalResult(report, _verbose) {
const { total, passed } = assertionSummary(report);
if (report.error) {
return `${icons.error} ${ansiBold(report.name)} ${dim(duration(report.duration))} ${ansi("red", `ERROR: ${report.error}`)}`;
}
const status = report.pass ? icons.pass : icons.fail;
const name = report.pass ? ansiBold(report.name) : ansiBoldColor("red", report.name);
const assertText = total > 0 ? dim(`${passed}/${total} assertions`) : dim("no assertions");
const turns = report.turns.length > 1 ? ` ${dim("turns:")} ${turnSummaryIcons(report.turns)}` : "";
return `${status} ${name} ${dim(duration(report.duration))} ${assertText}${turns}`;
}
function renderOutcomeAssertions(report) {
if (!report.outcomeAssertions?.length)
return [];
const lines = ["", `${INDENT}${INDENT}${dim("Outcome assertions:")}`];
for (const a of report.outcomeAssertions) {
if (a.pass) {
lines.push(`${INDENT}${INDENT} ${icons.pass} ${dim(a.assertion)}`);
} else {
lines.push(`${INDENT}${INDENT} ${icons.fail} ${ansi("red", a.assertion)}`);
lines.push(`${INDENT}${INDENT} ${dim("expected:")} ${a.expected}`);
lines.push(`${INDENT}${INDENT} ${dim("actual:")} ${a.actual}`);
}
}
return lines;
}
function renderFailureDetails(report) {
const lines = [];
for (const turn of report.turns) {
const turnLabel = report.turns.length > 1 ? dim(`Turn ${turn.turnIndex + 1}: `) : "";
const turnStatus = turn.pass ? `${turnLabel}${icons.pass} ${dim("passed")}` : `${turnLabel}${icons.fail} ${ansi("red", "failed")}`;
lines.push(`${INDENT}${INDENT}${turnStatus}`);
if (!turn.pass) {
lines.push(`${INDENT}${INDENT}${dim("User:")} ${truncate(turn.userMessage, 80)}`);
lines.push(`${INDENT}${INDENT}${dim("Bot:")} ${truncate(turn.botResponse, 80)}`);
lines.push("");
for (const a of turn.assertions) {
if (a.pass) {
lines.push(`${INDENT}${INDENT} ${icons.pass} ${dim(a.assertion)}`);
} else {
lines.push(`${INDENT}${INDENT} ${icons.fail} ${ansi("red", a.assertion)}`);
lines.push(`${INDENT}${INDENT} ${dim("expected:")} ${a.expected}`);
lines.push(`${INDENT}${INDENT} ${dim("actual:")} ${a.actual}`);
}
}
}
}
lines.push(...renderOutcomeAssertions(report));
return lines.join(`
`);
}
function renderSingleTurn(turn, isMultiTurn) {
const lines = [];
const turnTime = dim(duration(turn.botDuration + turn.evalDuration));
if (isMultiTurn) {
lines.push(`${INDENT}${INDENT}${dim(`Turn ${turn.turnIndex + 1}`)} ${turnTime}`);
}
lines.push(`${INDENT}${INDENT}${dim("User:")} ${truncate(turn.userMessage, 80)}`);
lines.push(`${INDENT}${INDENT}${dim("Bot:")} ${truncate(turn.botResponse, 80)}`);
for (const a of turn.assertions) {
if (a.pass) {
lines.push(`${INDENT}${INDENT} ${icons.pass} ${dim(a.assertion)}`);
} else {
lines.push(`${INDENT}${INDENT} ${icons.fail} ${ansi("red", a.assertion)}`);
lines.push(`${INDENT}${INDENT} ${dim("expected:")} ${a.expected}`);
lines.push(`${INDENT}${INDENT} ${dim("actual:")} ${a.actual}`);
}
}
return lines;
}
function renderVerboseDetails(report) {
const lines = [];
const isMulti = report.turns.length > 1;
for (const turn of report.turns) {
lines.push(...renderSingleTurn(turn, isMulti));
if (turn !== report.turns[report.turns.length - 1])
lines.push("");
}
lines.push(...renderOutcomeAssertions(report));
return lines.join(`
`);
}
function renderSuiteSummary(reports) {
const total = reports.length;
const passed = reports.filter((r) => r.pass).length;
const failed = total - passed;
const totalTime = reports.reduce((t, r) => t + r.duration, 0);
const totalAssertions = reports.reduce((n, r) => {
const turnAssertions = r.turns.reduce((tn, t) => tn + t.assertions.length, 0);
const outcomeCount = r.outcomeAssertions?.length || 0;
return n + turnAssertions + outcomeCount;
}, 0);
const passedAssertions = reports.reduce((n, r) => {
const turnPassed = r.turns.reduce((tn, t) => tn + t.assertions.filter((a) => a.pass).length, 0);
const outcomePassed = r.outcomeAssertions?.filter((a) => a.pass).length || 0;
return n + turnPassed + outcomePassed;
}, 0);
const lines = [];
lines.push("");
lines.push(`${INDENT}${bar(passed, total)}`);
lines.push("");
const evalSummary = failed > 0 ? `${ansiBoldColor("green", String(passed))} passed ${dim("/")} ${ansiBoldColor("red", String(failed))} failed` : ansiBoldColor("green", `${passed} passed`);
const assertSummary = totalAssertions > 0 ? dim(`${passedAssertions}/${totalAssertions} assertions`) : "";
lines.push(`${INDENT}${ansiBold("Evals:")} ${evalSummary} ${dim("of")} ${total}`);
if (totalAssertions > 0) {
lines.push(`${INDENT}${ansiBold("Assertions:")} ${assertSummary}`);
}
lines.push(`${INDENT}${ansiBold("Duration:")} ${dim(duration(totalTime))}`);
return lines.join(`
`);
}
function renderRunSummaryRow(run, index) {
const date = new Date(run.timestamp);
const dateStr = date.toLocaleDateString("en-US", {
month: "short",
day: "numeric",
hour: "2-digit",
minute: "2-digit"
});
const status = run.failed > 0 ? ansi("red", `${run.failed} failed`) : run.total > 0 ? ansi("green", "all passed") : dim("empty");
const idShort = run.id.slice(0, 8);
return `${INDENT}${dim(`${index + 1}.`)} ${ansiBold(idShort)} ${dim(dateStr)} ${status} ${dim(`${run.passed}/${run.total}`)} ${dim(duration(run.duration))}`;
}
function renderRunDetail(run, verbose) {
const lines = [];
lines.push("");
lines.push(`${INDENT}${ansiBold("Run")} ${run.id}`);
lines.push(`${INDENT}${dim(new Date(run.timestamp).toLocaleString())}`);
lines.push("");
for (const report of run.evals) {
lines.push(`${INDENT}${renderEvalResult(report, verbose)}`);
if (!report.pass && !report.error) {
lines.push(renderFailureDetails(report));
lines.push("");
} else if (verbose && !report.error) {
lines.push(renderVerboseDetails(report));
lines.push("");
}
}
lines.push(renderSuiteSummary(run.evals));
lines.push("");
return lines.join(`
`);
}
async function adkEvalsRun(name, opts) {
ensureJsonOnlyFormat(opts.format);
const logger = createCliLogger({ format: opts.format });
const agentRoot = await findAgentRootOrFail(process.cwd());
const filter = {};
if (name)
filter.names = [name];
if (opts.tag)
filter.tags = [opts.tag];
if (opts.type)
filter.type = opts.type;
if (opts.prod) {
return adkEvalsRunProd(agentRoot, filter, opts, logger);
}
return adkEvalsRunDev(agentRoot, filter, opts, logger);
}
async function adkEvalsRunProd(agentRoot, filter, opts, logger) {
const context = await resolveCommandContext({
cwd: agentRoot,
target: "prod",
require: ["project", "credentials"]
});
const project = context.project;
const credentials = context.credentials;
const botId = project.agentInfo?.botId;
const workspaceId = project.agentInfo?.workspaceId ?? credentials.workspaceId;
if (!botId) {
logger.error("No production bot found. Deploy with `adk deploy` first.").result({
success: false,
error: "No production bot \u2014 run `adk deploy`"
});
process.exit(1);
}
const client = new Uk({
token: credentials.token,
botId,
workspaceId,
apiUrl: credentials.apiUrl || "https://api.botpress.cloud",
headers: { "x-multiple-integrations": "true" }
});
if (opts.format !== "json") {
logger.newline();
logger.info(`${INDENT}${ansiBold("adk evals --prod")} ${dim(`targeting ${botId.slice(0, 8)}\u2026`)}`);
logger.newline();
}
const setupSpinner = opts.format !== "json" ? createSpinner("Preparing eval workflow...") : null;
const { workflows: stale } = await client.listWorkflows({
statuses: ["pending", "in_progress", "listening", "paused"]
});
const staleEvals = stale.filter((w) => w.name === "builtin_eval_runner");
for (const w of staleEvals) {
try {
await client.updateWorkflow({ id: w.id, status: "cancelled" });
} catch {}
}
if (staleEvals.length > 0) {
setupSpinner?.update(`Cancelled ${staleEvals.length} stale workflow${staleEvals.length === 1 ? "" : "s"}, creating new one...`);
}
const hasFilter = Object.keys(filter).length > 0;
const timeoutAt = new Date(Date.now() + 60 * 60 * 1000).toISOString();
const { workflow } = await client.createWorkflow({
name: "builtin_eval_runner",
status: "pending",
input: {
...hasFilter ? { filter } : {},
runType: "manual",
...opts.judgeModel ? { judgeModel: opts.judgeModel } : {}
},
timeoutAt
});
const workflowId = workflow.id;
const staleMsg = staleEvals.length > 0 ? `, cancelled ${staleEvals.length} stale` : "";
setupSpinner?.stop(`${icons.pass} ${dim(`Workflow created: ${workflowId.slice(0, 8)}\u2026${staleMsg}`)}`);
const pollSpinner = opts.format !== "json" ? createSpinner("Waiting for eval run to complete...") : null;
const pollIntervalMs = 3000;
const deadline = Date.now() + 60 * 60 * 1000;
while (Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
let current;
try {
const response = await client.getWorkflow({ id: workflowId });
current = response.workflow;
} catch {
continue;
}
if (current.status === "completed") {
const output = current.output;
pollSpinner?.stop(`${icons.pass} ${dim("Eval run completed")}`);
if (opts.format === "json") {
logger.info("").result({ workflowId, status: "completed", ...output });
} else {
const passed = output?.passed ?? 0;
const failed = output?.failed ?? 0;
const total = output?.total ?? 0;
const dur = output?.duration ?? 0;
logger.newline();
logger.info(`${INDENT}${bar(passed, total)}`);
logger.newline();
const evalSummary = failed > 0 ? `${ansiBoldColor("green", String(passed))} passed ${dim("/")} ${ansiBoldColor("red", String(failed))} failed` : ansiBoldColor("green", `${passed} passed`);
logger.info(`${INDENT}${ansiBold("Evals:")} ${evalSummary} ${dim("of")} ${total}`);
logger.info(`${INDENT}${ansiBold("Duration:")} ${dim(duration(dur))}`);
logger.info(`${INDENT}${ansiBold("Run ID:")} ${dim(output?.runId ?? workflowId)}`);
logger.newline();
}
process.exit(output?.failed ? 1 : 0);
}
if (current.status === "failed" || current.status === "timed_out") {
pollSpinner?.stop(`${icons.error} ${ansi("red", `Eval workflow ${current.status}`)}`);
const reason = current.failureReason || current.status;
logger.error(`${INDENT}${reason}`).result({ workflowId, status: current.status, error: reason });
process.exit(1);
}
if (current.status === "running") {
pollSpinner?.update("Eval run in progress...");
}
}
pollSpinner?.stop(`${icons.error} ${ansi("red", "Timed out waiting for eval workflow")}`);
process.exit(1);
}
async function adkEvalsRunDev(agentRoot, filter, opts, logger) {
const serverUrl = opts.server || getDevServerUrl();
const evalsDir = path.join(agentRoot, "evals");
const project = await AgentProject.load(agentRoot);
const botEvalOptions = {
idleTimeout: project.config?.evals?.idleTimeout,
judgePassThreshold: project.config?.evals?.judgePassThreshold,
judgeModel: opts.judgeModel ?? project.config?.evals?.judgeModel
};
let evals = await loadEvalsFromDir(evalsDir);
if (evals.length === 0) {
logger.warn(`No eval files found in ${evalsDir}/`).result({ success: false, error: `No eval files found in ${evalsDir}/` });
process.exit(0);
}
evals = filterEvals(evals, filter);
if (evals.length === 0) {
const message = filter.names?.[0] ? `No eval found with name "${filter.names[0]}"` : "No evals match the given filters.";
logger.error(message).result({ success: false, error: message });
process.exit(1);
}
if (opts.format === "json") {
let server2;
try {
server2 = await ensureDevServer(serverUrl, agentRoot);
} catch (err) {
logger.error(err.message).result({ success: false, error: err.message });
process.exit(1);
}
try {
const jsonPrompt = async () => {
throw new AdkError({
code: "EVAL_RUN_FAILED",
message: [
"The `chat` integration is not installed on this bot \u2014 `adk evals` requires it.",
"",
"To fix:",
" 1. Install it: adk integrations add chat",
" 2. Redeploy: adk dev (or: adk deploy)"
].join(`
`)
});
};
await ensureChatIntegrationReady(agentRoot, server2.credentials.devBotId, {
promptForInstall: jsonPrompt,
project,
client: createDevServerClient(server2.credentials)
});
} catch (err) {
logger.error(err.message).result({ success: false, error: err.message });
server2.cleanup();
process.exit(1);
}
try {
const bpClient = new Uk({
token: server2.credentials.token,
botId: server2.credentials.devBotId,
apiUrl: server2.credentials.apiUrl,
...server2.credentials.workspaceId ? { workspaceId: server2.credentials.workspaceId } : {}
});
const config = {
client: bpClient,
botId: server2.credentials.devBotId,
agentPath: agentRoot,
devServerUrl: server2.serverUrl,
devServerHeaders: server2.devServerHeaders,
chatClient: getChatClient(),
evalOptions: botEvalOptions,
onException: (error, properties) => telemetry_default.captureException(error, properties)
};
const report = await runEvalSuite(config, filter);
try {
await saveRunResult(agentRoot, report);
} catch {}
logger.info("Eval suite completed").result(report);
process.exit(report.evals.every((r) => r.pass) ? 0 : 1);
} finally {
server2.cleanup();
}
}
logger.newline();
logger.info(`${INDENT}${ansiBold("adk evals")} ${dim(`running ${evals.length} eval${evals.length === 1 ? "" : "s"}`)}`);
logger.newline();
const spinner = createSpinner(`Connecting to dev server at ${serverUrl}...`);
let server;
try {
server = await ensureDevServer(serverUrl, agentRoot);
if (server.lightweight) {
spinner.stop(`${icons.pass} ${dim("Lightweight server started (no running adk dev detected)")}`);
} else {
spinner.stop(`${icons.pass} ${dim("Connected to dev server")}`);
}
} catch (err) {
spinner.stop(`${icons.error} ${ansi("red", `Failed to connect: ${err.message}`)}`);
process.exit(1);
}
const chatSpinner = createSpinner("Checking chat integration...");
let promptShown = false;
try {
const interactivePrompt = async (message) => {
chatSpinner.stop(`${icons.run} ${dim("Chat integration not found")}`);
promptShown = true;
const { createInterface } = await import("readline/promises");
const rl = createInterface({ input: process.stdin, output: process.stderr });
try {
const answer = await rl.question(`${INDENT}${message}`);
const normalized = answer.trim().toLowerCase();
return normalized === "y" || normalized === "yes";
} finally {
rl.close();
}
};
const chatResult = await ensureChatIntegrationReady(agentRoot, server.credentials.devBotId, {
promptForInstall: interactivePrompt,
project,
client: createDevServerClient(server.credentials)
});
if (promptShown) {
logger.info(`${INDENT}${icons.pass} ${dim("Chat integration installed and enabled")}`);
} else if (chatResult.syncedToBot) {
chatSpinner.stop(`${icons.pass} ${dim("Chat integration synced")}`);
} else {
chatSpinner.stop(`${icons.pass} ${dim("Chat integration ready")}`);
}
} catch (err) {
if (!promptShown) {
chatSpinner.stop(`${icons.error} ${ansi("red", err.message)}`);
} else {
logger.error(`${INDENT}${icons.error} ${ansi("red", err.message)}`);
}
server.cleanup();
process.exit(1);
}
logger.newline();
try {
const reports = [];
let currentSpinner = null;
const bpClient = new Uk({
token: server.credentials.token,
botId: server.credentials.devBotId,
apiUrl: server.credentials.apiUrl,
...server.credentials.workspaceId ? { workspaceId: server.credentials.workspaceId } : {}
});
const config = {
client: bpClient,
botId: server.credentials.devBotId,
agentPath: agentRoot,
devServerUrl: server.serverUrl,
devServerHeaders: server.devServerHeaders,
chatClient: getChatClient(),
evalOptions: botEvalOptions,
logger: {
info: (msg) => logger.info(`${INDENT}${icons.dot} ${dim(msg)}`),
warn: (msg) => logger.warn(`${INDENT}${icons.error} ${msg}`),
error: (msg) => logger.error(`${INDENT}${icons.error} ${msg}`)
},
onException: (error, properties) => telemetry_default.captureException(error, properties),
onProgress: (event) => {
switch (event.type) {
case "eval_start": {
if (event.index > 0) {
const cols = process.stdout.columns || 80;
logger.info(`${INDENT}${dim("\u2500".repeat(Math.max(cols - INDENT.length, 20)))}`);
logger.newline();
}
const turnCount = event.totalTurns;
currentSpinner = createSpinner(`${ansiBold(event.evalName)} ${dim(`(${turnCount} turn${turnCount === 1 ? "" : "s"})`)}`);
break;
}
case "turn_start": {
if (currentSpinner && event.totalTurns > 1) {
currentSpinner.update(`${ansiBold(event.evalName)} ${dim(`(turn ${event.turnIndex + 1}/${event.totalTurns})`)}`);
}
break;
}
case "turn_complete": {
if (currentSpinner && opts.verbose) {
const lines = renderSingleTurn(event.turnReport, event.totalTurns > 1);
currentSpinner.printAbove(lines.join(`
`));
}
break;
}
case "eval_complete": {
const report = event.report;
reports.push(report);
if (currentSpinner) {
currentSpinner.stop(renderEvalResult(report, !!opts.verbose));
}
if (opts.verbose && !report.error) {
const outcomeLines = renderOutcomeAssertions(report);
if (outcomeLines.length > 0) {
logger.info(outcomeLines.join(`
`));
}
logger.newline();
} else if (!report.pass && !report.error) {
logger.info(renderFailureDetails(report));
logger.newline();
}
break;
}
}
}
};
const suiteReport = await runEvalSuite(config, filter);
try {
await saveRunResult(agentRoot, suiteReport);
} catch {}
logger.info(renderSuiteSummary(reports));
logger.newline();
if (server.lightweight) {
logger.info(`${INDENT}${icons.pass} ${dim("Lightweight server stopped")}`);
logger.newline();
logger.info(`${INDENT}To inspect results in a browser, run: adk dev`, "gray");
logger.info(`${INDENT}To view this run in the terminal: adk evals runs ${suiteReport.id}`, "gray");
logger.newline();
}
process.exit(suiteReport.evals.every((r) => r.pass) ? 0 : 1);
} finally {
server.cleanup();
}
}
async function adkEvalsRuns(runId, opts) {
ensureJsonOnlyFormat(opts.format);
const logger = createCliLogger({ format: opts.format });
const agentRoot = await findAgentRootOrFail(process.cwd());
if (opts.latest) {
const run = await getLatestRun(agentRoot);
if (!run) {
logger.warn("No eval runs found.");
process.exit(0);
}
logger.info(renderRunDetail(run, !!opts.verbose)).result(run);
return;
}
if (runId) {
const run = await loadRunResult(agentRoot, runId);
if (!run) {
logger.error(`No run found matching "${runId}"`);
process.exit(1);
}
logger.info(renderRunDetail(run, !!opts.verbose)).result(run);
return;
}
const limit = parseInt(opts.limit || "10");
const runs = await listRunResults(agentRoot, limit);
if (runs.length === 0) {
logger.warn("No eval runs found. Run `adk evals` to create one.");
process.exit(0);
}
const summaries = runs.map((r) => ({
id: r.id,
timestamp: r.timestamp,
passed: r.passed,
failed: r.failed,
total: r.total,
duration: r.duration
}));
logger.info(`${INDENT}${ansiBold("Eval Runs")} ${dim(`(${runs.length} most recent)`)}`).result(summaries);
logger.newline();
for (let i = 0;i < runs.length; i++) {
logger.info(renderRunSummaryRow(runs[i], i));
}
logger.newline();
logger.info(`${INDENT}Run \`adk evals runs <id>\` to see details, or \`adk evals runs --latest\` for the latest.`, "gray");
logger.newline();
}
export {
adkEvalsRuns,
adkEvalsRun
};