@botpress/adk-cli
Version:
Command-line interface for the Botpress Agent Development Kit (ADK)
2,705 lines • 91.3 kB
JavaScript
// @bun
import {
isFeatureEnabled
} from "./chunk-68mqsf42.js";
import {
buildDevServerHeaders
} from "./chunk-ndxsgd72.js";
import {
AgentMapSnapshotNotPublishedError,
prodAgentMapSnapshotService
} from "./chunk-r72adjnh.js";
import {
telemetry_default
} from "./chunk-kwmsaz7n.js";
import {
createCliLogger
} from "./chunk-gzwt1qdr.js";
import {
AdkError,
ConfigManager,
ConfigWriter,
DEPLOYED_AGENT_MANIFEST_FILE_KEY,
DEPLOYED_AGENT_MANIFEST_TAGS,
deployedAgentManifestSchema,
getChatClient,
getProjectClient
} from "./chunk-p0hjqn4r.js";
import {
ne
} from "./chunk-6w0knnta.js";
import {
EVAL_MANIFEST_SCHEMA_VERSION,
EVAL_MANIFEST_TAGS,
runEvalSuite
} from "./chunk-t76d8fxx.js";
import {
Uk
} from "./chunk-3xrpxgq4.js";
// src/server/utils/cors.ts
var LOCALHOST_PATTERN = /^https?:\/\/(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/;
function isAllowedOrigin(origin) {
if (!origin)
return false;
return LOCALHOST_PATTERN.test(origin);
}
function getCorsHeaders(req) {
const origin = req.headers.get("origin");
if (!isAllowedOrigin(origin))
return {};
return {
"Access-Control-Allow-Origin": origin,
"Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS",
"Access-Control-Allow-Headers": "Content-Type",
"Access-Control-Expose-Headers": "X-ADK-DevConsole",
Vary: "Origin"
};
}
function handleCorsPreflightResponse(req) {
return new Response(null, {
headers: getCorsHeaders(req),
status: 204
});
}
// src/server/utils/timing.ts
var timingLogger = createCliLogger({ tag: "timing" });
var eventLoopLogger = createCliLogger({ tag: "event-loop" });
var DEFAULT_SLOW_REQUEST_MS = 1000;
var DEFAULT_EVENT_LOOP_LAG_MS = 500;
var EVENT_LOOP_SAMPLE_MS = 1000;
function thresholdFromEnv(name, fallback) {
const raw = process.env[name];
if (!raw)
return fallback;
const value = Number(raw);
return Number.isFinite(value) && value >= 0 ? value : fallback;
}
var slowRequestThresholdMs = thresholdFromEnv("ADK_DEV_SLOW_REQUEST_MS", DEFAULT_SLOW_REQUEST_MS);
var eventLoopLagThresholdMs = thresholdFromEnv("ADK_DEV_EVENT_LOOP_LAG_MS", DEFAULT_EVENT_LOOP_LAG_MS);
async function withRequestTiming(label, operation) {
const start = performance.now();
try {
return await operation();
} finally {
const duration = performance.now() - start;
if (duration >= slowRequestThresholdMs) {
timingLogger.warn(`${label} took ${Math.round(duration)}ms`);
}
}
}
function startEventLoopLagMonitor(label) {
let previous = performance.now();
const interval = setInterval(() => {
const now = performance.now();
const lag = now - previous - EVENT_LOOP_SAMPLE_MS;
previous = now;
if (lag >= eventLoopLagThresholdMs) {
eventLoopLogger.warn(`${label} blocked for ${Math.round(lag)}ms`);
}
}, EVENT_LOOP_SAMPLE_MS);
interval.unref?.();
return () => clearInterval(interval);
}
// src/server/config.ts
import { EventEmitter } from "events";
var serverConfig;
var serverStartTime;
var devCommandStatus = "building";
var latestWorkerStats = null;
var agent0RuntimeClient = null;
var devBotRuntimeState = {
running: false,
port: null,
url: null,
botId: null
};
function getServerConfig() {
return serverConfig;
}
function setServerConfig(config) {
serverConfig = config;
}
function getServerStartTime() {
return serverStartTime;
}
function setServerStartTime(time) {
serverStartTime = time;
}
function getDevCommandStatus() {
return devCommandStatus;
}
function setDevCommandStatus(status) {
devCommandStatus = status;
}
function getLatestWorkerStats() {
return latestWorkerStats;
}
function updateWorkerStats(stats) {
latestWorkerStats = stats;
}
function getAgent0RuntimeClient() {
return agent0RuntimeClient;
}
function setAgent0RuntimeClient(client) {
agent0RuntimeClient = client;
}
var activeEnvironment = "dev";
function getActiveEnvironment() {
return activeEnvironment;
}
function setActiveEnvironment(env) {
activeEnvironment = env;
}
function getDevBotRuntimeState() {
return devBotRuntimeState;
}
function setDevBotRuntimeState(state) {
devBotRuntimeState = {
...devBotRuntimeState,
...state
};
}
function resetDevBotRuntimeState() {
devBotRuntimeState = {
running: false,
port: null,
url: null,
botId: null
};
}
var serverEvents = new EventEmitter;
function emitSecretsValuesChanged() {
serverEvents.emit("secrets:values-changed");
}
function onSecretsValuesChanged(listener) {
serverEvents.on("secrets:values-changed", listener);
return () => serverEvents.off("secrets:values-changed", listener);
}
function emitProjectReloaded() {
serverEvents.emit("project:reloaded");
}
function onProjectReloaded(listener) {
serverEvents.on("project:reloaded", listener);
return () => serverEvents.off("project:reloaded", listener);
}
// src/server/utils/responses.ts
var JSON_HEADERS = {
"Content-Type": "application/json"
};
function jsonResponse(data, status = 200) {
return new Response(JSON.stringify(data), {
status,
headers: JSON_HEADERS
});
}
function errorResponse(error, message, status = 500) {
return jsonResponse({ error, message }, status);
}
function successResponse(data) {
return jsonResponse(data, 200);
}
// src/server/prod-agent-metadata.ts
var MANIFEST_CACHE_TTL_MS = 15000;
var AGENT_NOT_DEPLOYED_MESSAGE = "Agent has not been deployed yet. Deploy with `adk deploy` to publish the production target.";
class ProdManifestService {
cache = new Map;
async getManifest(target) {
const cacheKey = this.getCacheKey(target);
const cached = this.cache.get(cacheKey);
if (cached && Date.now() - cached.fetchedAt < MANIFEST_CACHE_TTL_MS) {
return cached.manifest;
}
const client = this.makeClient(target);
const file = await this.findManifestFile(client);
if (!file.url) {
throw new AdkError({
code: "PROD_METADATA_INVALID",
message: "Deployed ADK metadata file does not have a download URL."
});
}
const response = await fetch(file.url);
if (!response.ok) {
throw new AdkError({
code: "PROD_METADATA_INVALID",
message: `Failed to download deployed ADK metadata: ${response.status} ${response.statusText}`
});
}
const manifest = parseDeployedAgentManifest(await response.text());
this.cache.set(cacheKey, { fetchedAt: Date.now(), manifest });
return manifest;
}
clear(target) {
if (!target) {
this.cache.clear();
return;
}
this.cache.delete(this.getCacheKey(target));
}
async findManifestFile(client) {
try {
const { file: file2 } = await client.getFile({ id: DEPLOYED_AGENT_MANIFEST_FILE_KEY });
return file2;
} catch {}
const { files } = await client.listFiles({ tags: { ...DEPLOYED_AGENT_MANIFEST_TAGS } });
const file = files.find((candidate) => candidate.key === DEPLOYED_AGENT_MANIFEST_FILE_KEY) ?? files[0];
if (!file) {
throw new AdkError({
code: "AGENT_NOT_DEPLOYED",
message: AGENT_NOT_DEPLOYED_MESSAGE,
expected: true
});
}
return file;
}
makeClient(target) {
return new Uk({
token: target.token,
apiUrl: target.apiUrl,
workspaceId: target.workspaceId,
botId: target.botId,
headers: { "x-multiple-integrations": "true" }
});
}
getCacheKey(target) {
return `${target.apiUrl}:${target.workspaceId}:${target.botId}`;
}
}
class ProdAgentMetadataService {
manifestService;
constructor(manifestService = new ProdManifestService) {
this.manifestService = manifestService;
}
async getAgentDefinition(target) {
const manifest = await this.manifestService.getManifest(target);
return manifestToAgentDefinition(manifest, target);
}
}
var prodAgentMetadataService = new ProdAgentMetadataService;
function getLocalProdMetadataTarget(config) {
const token = config.credentials.token;
const apiUrl = config.credentials.apiUrl;
const workspaceId = config.project?.agentInfo?.workspaceId ?? config.credentials.workspaceId;
const botId = config.project?.agentInfo?.botId ?? config.credentials.prodBotId;
if (!token || !apiUrl || !workspaceId || !botId) {
return null;
}
return {
token,
apiUrl,
workspaceId,
botId,
botName: config.project?.config?.name,
agentPath: config.agentPath,
devBotId: config.project?.agentInfo?.devId ?? config.credentials.devBotId
};
}
function manifestToAgentDefinition(manifest, target) {
return {
name: manifest.agent.name ?? target.botName ?? "unknown",
version: "0.0.0",
description: manifest.agent.description,
path: target.agentPath,
agentInfo: {
botId: target.botId,
workspaceId: target.workspaceId,
apiUrl: target.apiUrl,
...target.devBotId ? { devId: target.devBotId } : {}
},
workflows: manifest.primitives.workflows.map(toWorkflowDefinition),
actions: manifest.primitives.actions.map(toDefinition),
tables: manifest.primitives.tables.map(toDefinition),
triggers: manifest.primitives.triggers.map(toDefinition),
conversations: manifest.primitives.conversations.map(toDefinition),
knowledge: manifest.primitives.knowledge.map(toDefinition)
};
}
function toWorkflowDefinition(primitive) {
return {
...primitive.definition,
path: primitive.source.path,
export: primitive.source.exportName
};
}
function toDefinition(primitive) {
return primitive.definition;
}
function parseDeployedAgentManifest(content) {
let value;
try {
value = JSON.parse(content);
} catch {
throw new AdkError({
code: "PROD_METADATA_INVALID",
message: "Deployed ADK metadata is not valid JSON."
});
}
const result = deployedAgentManifestSchema.safeParse(value);
if (!result.success) {
throw new AdkError({
code: "PROD_METADATA_INVALID",
message: `Invalid deployed ADK metadata: ${result.error.errors.map((e) => e.message).join(", ")}`
});
}
return result.data;
}
// src/server/utils/validation.ts
function parseEnv(req) {
const raw = new URL(req.url).searchParams.get("env");
if (raw !== "dev" && raw !== "prod")
return null;
return raw;
}
function validateProject() {
const serverConfig2 = getServerConfig();
if (!serverConfig2.project) {
return errorResponse("No project loaded", "Agent project not available", 500);
}
return null;
}
function validateCredentials() {
const serverConfig2 = getServerConfig();
if (!serverConfig2.credentials?.token) {
return errorResponse("No credentials configured", "Bot credentials are required", 401);
}
return null;
}
function validateBotId(environment) {
const environment_ = environment ?? getActiveEnvironment();
const serverConfig2 = getServerConfig();
const devId = serverConfig2.project?.agentInfo?.devId;
const prodBotId = serverConfig2.project?.agentInfo?.botId;
const targetBotId = environment_ === "prod" ? prodBotId : devId;
if (!targetBotId) {
const targetFile = environment_ === "prod" ? "agent.json" : "agent.local.json";
return errorResponse("No bot ID configured", `No ${environment_ === "prod" ? "production" : "development"} bot ID found in ${targetFile}`, 400);
}
return null;
}
function validateMethod(req, allowedMethod) {
if (req.method !== allowedMethod) {
return errorResponse("Method not allowed", `Only ${allowedMethod} requests are allowed`, 405);
}
return null;
}
function validateProjectAndCredentials() {
return validateProject() || validateCredentials();
}
function getTargetBotId(environment) {
const environment_ = environment ?? getActiveEnvironment();
const serverConfig2 = getServerConfig();
const devId = serverConfig2.project?.agentInfo?.devId;
const prodBotId = serverConfig2.project?.agentInfo?.botId;
return environment_ === "prod" ? prodBotId || null : devId || null;
}
function getScopedServerCredentials(botId) {
const credentials = getServerConfig().credentials;
return {
token: credentials.token,
apiUrl: credentials.apiUrl || "https://api.botpress.cloud",
...credentials.workspaceId ? { workspaceId: credentials.workspaceId } : {},
botId
};
}
// src/server/handlers/config-vars.ts
var { transforms } = ne;
var logger = createCliLogger({ tag: "config-vars" });
function getJsonSchemaType(prop) {
if (typeof prop !== "object")
return;
return Array.isArray(prop.type) ? prop.type[0] : prop.type;
}
function getDevContext() {
const serverConfig2 = getServerConfig();
const project = serverConfig2.project;
if (!project)
return null;
const schema = project.config?.configuration?.schema;
if (!schema)
return null;
const botId = getTargetBotId("dev");
if (!botId)
return null;
return { botId, schema };
}
async function fetchCloudConfiguration(botId, target) {
const creds = target ?? getServerConfig().credentials;
if (!creds?.token || !creds.apiUrl || !creds.workspaceId)
return null;
const client = await getProjectClient({
credentials: {
token: creds.token,
apiUrl: creds.apiUrl,
workspaceId: creds.workspaceId,
botId
},
botId
});
const { bot } = await client.getBot({ id: botId });
return {
schema: bot.configuration?.schema,
data: bot.configuration?.data ?? {}
};
}
function createConfigManager(botId, target) {
const creds = target ?? getServerConfig().credentials;
if (!creds?.token || !creds.apiUrl) {
return new ConfigManager(botId);
}
return new ConfigManager(botId, {
credentials: {
token: creds.token,
apiUrl: creds.apiUrl,
...creds.workspaceId ? { workspaceId: creds.workspaceId } : {},
botId
}
});
}
function getServerProdConfigTarget(prodBotId) {
const creds = getServerConfig().credentials;
if (!creds?.token || !creds.apiUrl || !creds.workspaceId)
return null;
return {
token: creds.token,
apiUrl: creds.apiUrl,
workspaceId: creds.workspaceId,
botId: prodBotId
};
}
function hasCloudConfigurationSchema(schema) {
return !!schema && Object.keys(schema).length > 0;
}
function reconstructZodFromCloud(cloudSchema) {
try {
const reconstructed = transforms.fromJSONSchema(cloudSchema);
return reconstructed;
} catch (error) {
logger.warn(`Failed to reconstruct Zod schema from cloud JSON schema: ${error instanceof Error ? error.message : String(error)}`);
return null;
}
}
async function handleGetConfigVariables(req) {
const env = parseEnv(req);
if (!env) {
return errorResponse("Invalid environment", 'env query param must be "dev" or "prod"', 400);
}
if (env === "dev") {
const ctx = getDevContext();
if (!ctx) {
return successResponse({
fields: [],
valid: true,
missing: [],
source: "local"
});
}
const configManager = createConfigManager(ctx.botId);
const [fields, validation] = await Promise.all([
configManager.describeSchema(ctx.schema),
configManager.validate(ctx.schema)
]);
return successResponse({
fields,
valid: validation.valid,
missing: validation.missing,
source: "local"
});
}
const prodBotId = getTargetBotId("prod");
if (!prodBotId) {
return errorResponse("No bot ID configured", "No production bot ID found in agent.json", 400);
}
const target = getServerProdConfigTarget(prodBotId);
if (!target) {
return errorResponse("No credentials configured", "Bot credentials are required", 401);
}
return handleGetProdConfigVariables(target);
}
async function handlePutConfigVariables(req) {
const env = parseEnv(req);
if (!env) {
return errorResponse("Invalid environment", 'env query param must be "dev" or "prod"', 400);
}
let body;
try {
body = await req.json();
} catch {
return errorResponse("Invalid JSON", "Request body must be valid JSON", 400);
}
if (env === "dev") {
const ctx = getDevContext();
if (!ctx) {
return errorResponse("No configuration schema", "No configuration schema defined in agent.config.ts", 400);
}
const configManager = createConfigManager(ctx.botId);
const results = {};
for (const [key, value] of Object.entries(body)) {
results[key] = await configManager.setWithValidation(key, value, ctx.schema);
}
return successResponse({ results, env });
}
const prodBotId = getTargetBotId("prod");
if (!prodBotId) {
return errorResponse("No bot ID configured", "No production bot ID found in agent.json", 400);
}
const target = getServerProdConfigTarget(prodBotId);
if (!target) {
return errorResponse("No credentials configured", "Bot credentials are required", 401);
}
return setProdConfigVariables(target, body);
}
async function handleGetProdConfigVariables(target) {
try {
const cloud = await fetchCloudConfiguration(target.botId, target);
if (!cloud) {
return errorResponse("No credentials configured", "Bot credentials are required", 401);
}
if (!hasCloudConfigurationSchema(cloud.schema)) {
return successResponse({
fields: [],
valid: true,
missing: [],
source: "cloud",
deployed: true,
schemaMissing: true
});
}
const zodSchema = reconstructZodFromCloud(cloud.schema);
if (!zodSchema) {
return errorResponse("Failed to parse deployed schema", "Deployed configuration schema could not be reconstructed", 500);
}
const configManager = createConfigManager(target.botId, target);
const [fields, validation] = await Promise.all([
configManager.describeSchema(zodSchema),
configManager.validate(zodSchema)
]);
return successResponse({
fields,
valid: validation.valid,
missing: validation.missing,
source: "cloud",
deployed: true
});
} catch (error) {
return errorResponse("Failed to load prod configuration", error instanceof Error ? error.message : "Unknown error", 500);
}
}
async function handlePutProdConfigVariables(req, target) {
let body;
try {
body = await req.json();
} catch {
return errorResponse("Invalid JSON", "Request body must be valid JSON", 400);
}
return setProdConfigVariables(target, body);
}
async function setProdConfigVariables(target, body) {
let cloud;
try {
cloud = await fetchCloudConfiguration(target.botId, target);
} catch (error) {
return errorResponse("Failed to load prod configuration", error instanceof Error ? error.message : "Unknown error", 500);
}
if (!cloud) {
return errorResponse("No credentials configured", "Bot credentials are required", 401);
}
if (!hasCloudConfigurationSchema(cloud.schema)) {
return errorResponse("Deployed configuration has no schema", "Run `adk deploy` to publish a configuration schema first.", 400);
}
const zodSchema = reconstructZodFromCloud(cloud.schema);
if (!zodSchema) {
return errorResponse("Failed to parse deployed schema", "Deployed configuration schema could not be reconstructed", 500);
}
const configManager = createConfigManager(target.botId, target);
const results = {};
for (const [key, value] of Object.entries(body)) {
results[key] = await configManager.setWithValidation(key, value, zodSchema);
}
return successResponse({ results, env: "prod" });
}
async function handlePatchConfigSchema(req) {
const env = parseEnv(req);
if (!env) {
return errorResponse("Invalid environment", 'env query param must be "dev" or "prod"', 400);
}
if (env === "prod") {
return errorResponse("schema_mutation_requires_deploy", "Prod schema changes must go through `adk deploy`.", 409);
}
const serverConfig2 = getServerConfig();
if (!serverConfig2.agentPath) {
return errorResponse("No agent path", "Agent path not configured", 400);
}
let updates;
try {
updates = await req.json();
} catch {
return errorResponse("Invalid JSON", "Request body must be valid JSON array", 400);
}
if (!Array.isArray(updates)) {
return errorResponse("Invalid body", "Request body must be an array", 400);
}
try {
const configWriter = new ConfigWriter(serverConfig2.agentPath);
await configWriter.updateConfiguration(updates);
return successResponse({ success: true });
} catch (error) {
return errorResponse("Update failed", error instanceof Error ? error.message : "Unknown error", 500);
}
}
async function handleGetConfigSchemaDiff() {
const serverConfig2 = getServerConfig();
const devZod = serverConfig2.project?.config?.configuration?.schema;
const prodBotId = getTargetBotId("prod");
if (!prodBotId) {
return successResponse({
deployed: false,
schemaMissing: false,
onlyInDev: [],
onlyInProd: [],
typeMismatches: []
});
}
let devSchema = { type: "object", properties: {}, required: [] };
if (devZod) {
try {
devSchema = devZod.toJSONSchema();
} catch (error) {
return errorResponse("Failed to serialise dev configuration schema", error instanceof Error ? error.message : "Unknown error", 500);
}
}
let cloud;
try {
cloud = await fetchCloudConfiguration(prodBotId);
} catch (error) {
return errorResponse("Failed to load prod configuration", error instanceof Error ? error.message : "Unknown error", 500);
}
if (!cloud) {
return errorResponse("No credentials configured", "Bot credentials are required", 401);
}
const prodSchema = hasCloudConfigurationSchema(cloud.schema) ? cloud.schema : undefined;
const devProps = devSchema.properties ?? {};
const prodProps = prodSchema?.properties ?? {};
const devKeys = new Set(Object.keys(devProps));
const prodKeys = new Set(Object.keys(prodProps));
const onlyInDev = [...devKeys].filter((k) => !prodKeys.has(k));
const onlyInProd = [...prodKeys].filter((k) => !devKeys.has(k));
const typeMismatches = [];
for (const key of devKeys) {
if (!prodKeys.has(key))
continue;
const d = getJsonSchemaType(devProps[key]);
const p = getJsonSchemaType(prodProps[key]);
if (d !== p) {
typeMismatches.push({ key, devType: d, prodType: p });
}
}
return successResponse({
deployed: true,
schemaMissing: !prodSchema,
onlyInDev,
onlyInProd,
typeMismatches
});
}
// src/server/production-observability.ts
var PRODUCTION_OBSERVABILITY_FLAG = "enable_production_observability";
function isProductionObservabilityEnabled() {
return isFeatureEnabled(PRODUCTION_OBSERVABILITY_FLAG);
}
function productionObservabilityDisabledResponse() {
return errorResponse("Production observability disabled", "Production evals and traces are not available in this build.", 403);
}
// ../evals/dist/stores/index.js
import { Database } from "bun:sqlite";
import { existsSync as existsSync2, mkdirSync, readdirSync as readdirSync2, readFileSync, renameSync } from "fs";
import { join } from "path";
import { readdirSync, existsSync } from "fs";
import { resolve } from "path";
var Eval = class {
name;
description;
tags;
type;
setup;
conversation;
outcome;
options;
constructor(def) {
this.name = def.name;
this.conversation = def.conversation;
if (def.description !== undefined)
this.description = def.description;
if (def.tags !== undefined)
this.tags = def.tags;
if (def.type !== undefined)
this.type = def.type;
if (def.setup !== undefined)
this.setup = def.setup;
if (def.outcome !== undefined)
this.outcome = def.outcome;
if (def.options !== undefined)
this.options = def.options;
}
};
var AdkError2 = class extends Error {
static __IS_ADK_BASE_ERROR = true;
code;
expected;
details;
suggestion;
constructor(opts) {
super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined);
this.name = this.constructor.name;
this.code = opts.code;
this.expected = opts.expected ?? false;
if (opts.details !== undefined) {
this.details = opts.details;
}
if (opts.suggestion !== undefined) {
this.suggestion = opts.suggestion;
}
}
};
var EvalRunnerError = class extends AdkError2 {
};
function isEvalDefinition(value) {
return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
}
async function loadEvalFile(filePath) {
const absPath = resolve(filePath);
let mod;
try {
mod = await import(absPath);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
throw new EvalRunnerError({
code: "EVAL_LOAD_FAILED",
message: `Failed to load eval file ${filePath}: ${msg}
Make sure your eval file:
- Has no syntax or type errors
- Exports one or more \`new Eval({...})\` instances
- Has all dependencies installed (\`bun install\`)`,
expected: true,
cause: err
});
}
const results = [];
for (const [key, value] of Object.entries(mod)) {
if (key === "__esModule")
continue;
if (value instanceof Eval || isEvalDefinition(value)) {
results.push(value);
}
}
if (results.length === 0) {
throw new EvalRunnerError({
code: "EVAL_FILE_EMPTY",
message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
expected: true
});
}
return results;
}
async function loadEvalsFromDir(dirPath) {
const absDir = resolve(dirPath);
if (!existsSync(absDir)) {
return [];
}
const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
const evals = [];
for (const f of files) {
const defs = await loadEvalFile(`${absDir}/${f}`);
evals.push(...defs);
}
const seen = /* @__PURE__ */ new Set;
for (const e of evals) {
if (seen.has(e.name)) {
throw new EvalRunnerError({
code: "EVAL_DUPLICATE_NAME",
message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
expected: true,
details: { name: e.name }
});
}
seen.add(e.name);
}
return evals;
}
function filterEvals(evals, filter) {
if (!filter)
return evals;
return evals.filter((e) => {
if (filter.names && filter.names.length > 0) {
if (!filter.names.includes(e.name))
return false;
}
if (filter.tags && filter.tags.length > 0) {
if (!e.tags || !filter.tags.some((t) => e.tags.includes(t)))
return false;
}
if (filter.type) {
if (e.type !== filter.type)
return false;
}
return true;
});
}
function createDiskEvalLoader(evalsDir) {
return () => loadEvalsFromDir(evalsDir);
}
var errText = (err) => err instanceof Error ? err.message : String(err);
var SCHEMA_VERSION = "2";
var SCHEMA = `
CREATE TABLE IF NOT EXISTS eval_runs (
id TEXT PRIMARY KEY,
timestamp TEXT NOT NULL,
started_at INTEGER NOT NULL,
passed INTEGER NOT NULL,
failed INTEGER NOT NULL,
total INTEGER NOT NULL,
duration REAL NOT NULL,
bot_duration REAL NOT NULL DEFAULT 0,
eval_duration REAL NOT NULL DEFAULT 0,
aborted INTEGER NOT NULL DEFAULT 0,
filter_json TEXT,
eval_names TEXT NOT NULL DEFAULT '[]'
);
CREATE INDEX IF NOT EXISTS idx_eval_runs_started ON eval_runs(started_at DESC);
CREATE TABLE IF NOT EXISTS eval_reports (
run_id TEXT NOT NULL REFERENCES eval_runs(id) ON DELETE CASCADE,
seq INTEGER NOT NULL,
name TEXT NOT NULL,
started_at INTEGER NOT NULL,
pass INTEGER NOT NULL,
duration REAL NOT NULL,
type TEXT,
error TEXT,
report_json TEXT NOT NULL,
PRIMARY KEY (run_id, seq)
);
CREATE INDEX IF NOT EXISTS idx_eval_reports_name_time ON eval_reports(name, started_at DESC);
CREATE TABLE IF NOT EXISTS schema_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
`;
function aggregateTurnTimings(report) {
let botDuration = 0;
let evalDuration = 0;
for (const e of report.evals) {
for (const t of e.turns) {
botDuration += t.botDuration;
evalDuration += t.evalDuration;
}
}
return { botDuration, evalDuration };
}
function toEvalSummary(e) {
return {
name: e.name,
description: e.description,
tags: e.tags || [],
type: e.type || "capability",
turnCount: e.conversation.length,
hasOutcome: !!e.outcome
};
}
var LocalEvalStore = class {
db = null;
dbPath;
evalsDir;
insertRunStmt = null;
insertReportStmt = null;
_getActiveRunId;
_loadEvalDefinitions;
constructor(config) {
this.evalsDir = join(config.agentPath, ".adk", "evals");
this.dbPath = join(this.evalsDir, "evals.db");
this._getActiveRunId = config.getActiveRunId ?? (() => null);
this._loadEvalDefinitions = config.loadEvalDefinitions ?? createDiskEvalLoader(join(config.agentPath, "evals"));
}
async listEvals(filter) {
const allEvals = await this._loadEvalDefinitions();
const evals = filter ? filterEvals(allEvals, filter) : allEvals;
return evals.map(toEvalSummary);
}
async getEval(name) {
const allEvals = await this._loadEvalDefinitions();
return allEvals.find((e) => e.name === name) ?? null;
}
async createRun(_runType, _metadata) {
return crypto.randomUUID().replace(/-/g, "").slice(0, 26);
}
async addRunResults(_runId, _evalReport) {}
async completeRun(_runId, report) {
const db = this.getDb();
if (!db)
return;
db.run("BEGIN");
try {
this.insertRunReport(report);
db.run("COMMIT");
} catch (err) {
db.run("ROLLBACK");
console.error(`[local-eval-store] completeRun transaction failed (run ${report.id}): ${errText(err)}`);
throw err;
}
}
async loadRunResult(runId) {
const db = this.getDb();
if (!db)
return null;
const runRow = db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(runId);
if (!runRow)
return null;
const reportRows = db.prepare("SELECT * FROM eval_reports WHERE run_id = ? ORDER BY seq").all(runRow.id);
return this.assembleRun(runRow, reportRows);
}
async getLatestRun() {
const db = this.getDb();
if (!db)
return null;
const runRow = db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT 1").get();
if (!runRow)
return null;
const reportRows = db.prepare("SELECT * FROM eval_reports WHERE run_id = ? ORDER BY seq").all(runRow.id);
return this.assembleRun(runRow, reportRows);
}
async listRunSummaries(opts) {
const db = this.getDb();
if (!db)
return [];
const limit = opts?.limit ?? 50;
const sinceTs = opts?.since;
const rows = sinceTs !== undefined ? db.prepare("SELECT * FROM eval_runs WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?").all(sinceTs, limit) : db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT ?").all(limit);
return rows.map((r) => ({
id: r.id,
timestamp: r.timestamp,
passed: r.passed,
failed: r.failed,
total: r.total,
duration: r.duration,
botDuration: r.bot_duration,
evalDuration: r.eval_duration,
filter: r.filter_json ? JSON.parse(r.filter_json) : undefined,
evalNames: JSON.parse(r.eval_names),
aborted: r.aborted === 1
}));
}
async listEvalReportsByName(evalName, opts) {
const db = this.getDb();
if (!db)
return [];
const limit = opts?.limit ?? 20;
const sinceTs = opts?.since;
const baseSql = `
SELECT er.*,
r.timestamp AS run_timestamp,
r.total AS run_total,
r.filter_json AS run_filter_json
FROM eval_reports er
JOIN eval_runs r ON r.id = er.run_id
WHERE er.name = ?
`;
const rows = sinceTs !== undefined ? db.prepare(`${baseSql} AND er.started_at >= ? ORDER BY er.started_at DESC LIMIT ?`).all(evalName, sinceTs, limit) : db.prepare(`${baseSql} ORDER BY er.started_at DESC LIMIT ?`).all(evalName, limit);
return rows.map((row) => ({
runId: row.run_id,
timestamp: row.run_timestamp,
report: JSON.parse(row.report_json),
totalEvalsInRun: row.run_total,
filter: row.run_filter_json ? JSON.parse(row.run_filter_json) : undefined
}));
}
async listEvalReportsBulk(opts) {
const db = this.getDb();
if (!db)
return {};
const perEval = opts?.perEval ?? 20;
const sinceTs = opts?.since;
const sinceClause = sinceTs !== undefined ? "WHERE er.started_at >= ?" : "";
const sql = `
WITH ranked AS (
SELECT er.*, ROW_NUMBER() OVER (PARTITION BY name ORDER BY started_at DESC) AS rn
FROM eval_reports er
${sinceClause}
)
SELECT ranked.run_id, ranked.name, ranked.started_at, ranked.pass,
ranked.duration, ranked.type, ranked.error, ranked.report_json,
r.timestamp AS run_timestamp,
r.total AS run_total,
r.filter_json AS run_filter_json
FROM ranked
JOIN eval_runs r ON r.id = ranked.run_id
WHERE ranked.rn <= ?
ORDER BY ranked.name, ranked.started_at DESC
`;
const rows = sinceTs !== undefined ? db.prepare(sql).all(sinceTs, perEval) : db.prepare(sql).all(perEval);
const out = {};
for (const row of rows) {
const entry = {
runId: row.run_id,
timestamp: row.run_timestamp,
report: JSON.parse(row.report_json),
totalEvalsInRun: row.run_total,
filter: row.run_filter_json ? JSON.parse(row.run_filter_json) : undefined
};
const arr = out[row.name];
if (arr)
arr.push(entry);
else
out[row.name] = [entry];
}
return out;
}
async* watchRun(_signal, _options) {}
async getRunnerState() {
const runId = this._getActiveRunId();
return { running: runId !== null, runId };
}
async listRunResults(limit = 50, sinceTs) {
const db = this.getDb();
if (!db)
return [];
let runRows;
if (sinceTs !== undefined) {
runRows = db.prepare("SELECT * FROM eval_runs WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?").all(sinceTs, limit);
} else {
runRows = db.prepare("SELECT * FROM eval_runs ORDER BY started_at DESC LIMIT ?").all(limit);
}
if (runRows.length === 0)
return [];
const ids = runRows.map((r) => r.id);
const placeholders = ids.map(() => "?").join(",");
const reportRows = db.prepare(`SELECT * FROM eval_reports WHERE run_id IN (${placeholders}) ORDER BY seq`).all(...ids);
const byRun = /* @__PURE__ */ new Map;
for (const row of reportRows) {
const arr = byRun.get(row.run_id);
if (arr)
arr.push(row);
else
byRun.set(row.run_id, [row]);
}
return runRows.map((row) => this.assembleRun(row, byRun.get(row.id) ?? []));
}
close() {
try {
this.db?.close();
} catch {}
this.db = null;
this.insertRunStmt = null;
this.insertReportStmt = null;
}
getDbPath() {
return this.dbPath;
}
getDb() {
if (this.db)
return this.db;
mkdirSync(this.evalsDir, { recursive: true });
try {
this.db = new Database(this.dbPath);
this.db.run("PRAGMA journal_mode=WAL");
this.db.run("PRAGMA synchronous=NORMAL");
this.db.run("PRAGMA busy_timeout=5000");
this.db.run("PRAGMA foreign_keys=ON");
this.db.run(SCHEMA);
this.setMeta("schema_version", SCHEMA_VERSION);
this.insertRunStmt = this.db.prepare(`
INSERT OR REPLACE INTO eval_runs (
id, timestamp, started_at, passed, failed, total, duration,
bot_duration, eval_duration, aborted, filter_json, eval_names
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
this.insertReportStmt = this.db.prepare(`
INSERT OR REPLACE INTO eval_reports (
run_id, seq, name, started_at, pass, duration, type, error, report_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
this.importLegacyJsonRunsIfNeeded();
return this.db;
} catch (err) {
console.warn("[local-eval-store] Failed to initialize database:", err);
try {
this.db?.close();
} catch {}
try {
if (existsSync2(this.dbPath)) {
renameSync(this.dbPath, `${this.dbPath}.corrupt-${Date.now()}`);
}
} catch {}
this.db = null;
this.insertRunStmt = null;
this.insertReportStmt = null;
return null;
}
}
getMeta(key) {
const db = this.db;
if (!db)
return null;
const row = db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(key);
return row?.value ?? null;
}
setMeta(key, value) {
const db = this.db;
if (!db)
return;
db.prepare("INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)").run(key, value);
}
importLegacyJsonRunsIfNeeded() {
const db = this.db;
if (!db)
return;
if (this.getMeta("imported_legacy") === "1")
return;
const legacyDir = join(this.evalsDir, "runs");
if (!existsSync2(legacyDir)) {
this.setMeta("imported_legacy", "1");
return;
}
let files = [];
try {
files = readdirSync2(legacyDir).filter((f) => f.endsWith(".json"));
} catch (err) {
console.warn("[local-eval-store] Could not read legacy runs dir:", err);
return;
}
if (files.length === 0) {
this.setMeta("imported_legacy", "1");
return;
}
let imported = 0;
let skipped = 0;
db.run("BEGIN");
try {
for (const file of files) {
db.run("SAVEPOINT import_file");
try {
const raw = readFileSync(join(legacyDir, file), "utf-8");
const report = JSON.parse(raw);
this.insertRunReport(report);
db.run("RELEASE import_file");
imported++;
} catch (err) {
db.run("ROLLBACK TO import_file");
db.run("RELEASE import_file");
skipped++;
console.warn(`[local-eval-store] Skipping unreadable legacy run ${file}:`, err);
}
}
this.setMeta("imported_legacy", "1");
db.run("COMMIT");
} catch (err) {
db.run("ROLLBACK");
console.warn("[local-eval-store] Legacy import failed, will retry on next start:", err);
return;
}
try {
renameSync(legacyDir, join(this.evalsDir, "runs.json-migrated"));
} catch (err) {
console.warn("[local-eval-store] Could not rename legacy runs dir:", err);
}
console.log(`[local-eval-store] Imported ${imported} legacy eval run${imported === 1 ? "" : "s"}` + (skipped > 0 ? ` (${skipped} skipped)` : ""));
}
insertRunReport(report) {
if (!this.insertRunStmt || !this.insertReportStmt)
return;
const startedAt = Date.parse(report.timestamp);
const { botDuration, evalDuration } = aggregateTurnTimings(report);
this.insertRunStmt.run(report.id, report.timestamp, Number.isNaN(startedAt) ? 0 : startedAt, report.passed, report.failed, report.total, report.duration, botDuration, evalDuration, report.aborted ? 1 : 0, report.filter ? JSON.stringify(report.filter) : null, JSON.stringify(report.evals.map((e) => e.name)));
for (let i = 0;i < report.evals.length; i++) {
const evalReport = report.evals[i];
this.insertReportStmt.run(report.id, i, evalReport.name, Number.isNaN(startedAt) ? 0 : startedAt, evalReport.pass ? 1 : 0, evalReport.duration, evalReport.type ?? null, evalReport.error ?? null, JSON.stringify(evalReport));
}
}
assembleRun(runRow, reportRows) {
const evals = reportRows.map((r) => JSON.parse(r.report_json));
const report = {
id: runRow.id,
timestamp: runRow.timestamp,
evals,
passed: runRow.passed,
failed: runRow.failed,
total: runRow.total,
duration: runRow.duration
};
if (runRow.filter_json) {
report.filter = JSON.parse(runRow.filter_json);
}
if (runRow.aborted === 1) {
report.aborted = true;
}
return report;
}
};
var stores = /* @__PURE__ */ new Map;
function getLocalEvalStore(agentPath, getActiveRunId) {
let store = stores.get(agentPath);
if (!store) {
store = new LocalEvalStore({ agentPath, getActiveRunId });
stores.set(agentPath, store);
}
return store;
}
var OUTCOME_TURN_INDEX = -1;
function vortexEntryToEvalReport(entry) {
const outcomeAssertions = entry.results.filter((r) => r.turnIndex === OUTCOME_TURN_INDEX).map((r) => ({
assertion: r.graderName,
pass: r.passed,
expected: r.evidence?.expected ?? "",
actual: r.evidence?.actual ?? ""
}));
const resultsByTurn = /* @__PURE__ */ new Map;
for (const r of entry.results) {
if (r.turnIndex === OUTCOME_TURN_INDEX)
continue;
const existing = resultsByTurn.get(r.turnIndex) ?? [];
existing.push(r);
resultsByTurn.set(r.turnIndex, existing);
}
const turns = [...resultsByTurn.entries()].sort(([a], [b]) => a - b).map(([turnIndex, results]) => {
const assertions = results.map((r) => ({
assertion: r.graderName,
pass: r.passed,
expected: r.evidence?.expected ?? "",
actual: r.evidence?.actual ?? ""
}));
const first = results[0];
return {
turnIndex,
userMessage: first.userMessage,
botResponse: first.botResponse,
assertions,
pass: assertions.every((a) => a.pass),
botDuration: first.botDurationMs ?? 0,
evalDuration: first.graderDurationMs ?? 0
};
});
return {
name: entry.evalName,
description: entry.description || undefined,
type: entry.evalType,
tags: entry.tags,
turns,
outcomeAssertions,
pass: entry.passed ?? false,
duration: entry.durationMs ?? 0,
error: entry.error ?? undefined
};
}
async function mapWithConcurrency(items, concurrency, fn) {
const results = new Array(items.length);
let next = 0;
const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
while (next < items.length) {
const i = next++;
results[i] = await fn(items[i]);
}
});
await Promise.all(workers);
return results;
}
function turnToResultRows(turn) {
return graderResultsToRows(turn.assertions, {
turnIndex: turn.turnIndex,
userMessage: turn.userMessage,
botResponse: turn.botResponse,
botDurationMs: turn.botDuration,
graderDurationMs: turn.evalDuration
});
}
function outcomeToResultRows(assertions) {
return graderResultsToRows(assertions, {
turnIndex: OUTCOME_TURN_INDEX,
userMessage: "",
botResponse: "",
botDurationMs: 0,
graderDurationMs: 0
});
}
function graderResultsToRows(assertions, context) {
return assertions.map((assertion) => ({
...context,
graderName: assertion.assertion,
passed: assertion.pass,
evidence: { expected: assertion.expected, actual: assertion.actual }
}));
}
function runMatchesWatchOptions(run, options) {
if (options.runId && run.id !== options.runId)
return false;
if (options.workflowId && run.metadata?.workflowId !== options.workflowId)
return false;
return true;
}
function vortexRunToReport(run) {
const evals = run.entries.map(vortexEntryToEvalReport);
return {
id: run.id,
timestamp: run.createdAt,
evals,
passed: evals.filter((e) => e.pass).length,
failed: evals.filter((e) => !e.pass).length,
total: evals.length,
duration: evals.reduce((s, e) => s + e.duration, 0)
};
}
function vortexRunToSummary(run) {
const startMs = run.startedAt ? new Date(run.startedAt).getTime() : 0;
const endMs = run.completedAt ? new Date(run.completedAt).getTime() : 0;
if (run.entries) {
const evals = run.entries.map(vortexEntryToEvalReport);
return {
id: run.id,
timestamp: run.createdAt,
passed: evals.filter((e) => e.pass).length,
failed: evals.filter((e) => !e.pass).length,
total: evals.length,
duration: evals.reduce((s, e) => s + e.duration, 0),
botDuration: evals.reduce((s, e) => e.turns.reduce((ts, t) => ts + t.botDuration, s), 0),
evalDuration: evals.reduce((s, e) => e.turns.reduce((ts, t) => ts + t.evalDuration, s), 0),
evalNames: evals.map((e) => e.name),
aborted: false
};
}
return {
id: run.id,
timestamp: run.createdAt,
passed: 0,
failed: 0,
total: 0,
duration: startMs && endMs ? endMs - startMs : 0,
botDuration: 0,
evalDuration: 0,
evalNames: [],
aborted: false
};
}
var VortexEvalStore = class {
url;
botId;
workspaceId;
token;
_evalManifestId;
_loadEvalDefinitions;
constructor(config) {
this.url = config.url.replace(/\/$/, "");
this.botId = config.botId;
if (config.workspaceId)
this.workspaceId = config.workspaceId;
if (config.token)
this.token = config.token;
this._evalManifestId = config.evalManifestId;
this._loadEvalDefinitions = config.loadEvalDefinitions;
}
async listEvals(filter) {
const defs = await this.getEvalDefinitions();
const filtered = defs.filter((e) => {
if (filter?.names && !filter.names.includes(e.name))
return false;
if (filter?.type && e.type !== filter.type)
return false;
if (filter?.tags && !filter.tags.every((tag) => e.tags?.includes(tag)))
return false;
return true;
});
return filtered.map((e) => ({
name: e.name,
description: e.description,
tags: e.tags || [],
type: e.type || "capability",
turnCount: e.conversation.length,
hasOutcome: !!e.outcome
}));
}
async getEval(name) {
const defs = await this.getEvalDefinitions();
return defs.find((e) => e.name === name) ?? null;
}
async createRun(runType = "scheduled", metadata) {
if (!this.workspaceId) {
throw new Error("workspaceId is required to create Vortex eval runs");
}
const triggerType = runType === "manual" ? "manual" : "scheduled";
const res = await this.fetch(`/v1/evals/bot/${this.botId}/runs`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
evalManifestId: (typeof this._evalManifestId === "function" ? this._evalManifestId() : this._evalManifestId) ?? "",
workspaceId: this.workspaceId,
triggerType,
...metadata ? { metadata } : {}
})
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex create run failed: ${res.status} ${text}`);
}
const { id } = await res.json();
return id;
}
async addRunResults(runId, evalReport) {
const entry = {
evalName: evalReport.name,
evalType: evalReport.type ?? "capability",
description: evalReport.description ?? "",
tags: evalReport.tags ?? [],
passed: evalReport.pass,
durationMs: evalReport.duration,
error: evalReport.error,
results: [...evalReport.turns.flatMap(turnToResultRows), ...outcomeToResultRows(evalReport.outcomeAssertions)]
};
const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ entries: [entry] })
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex ingest entries failed: ${res.status} ${text}`);
}
}
async completeRun(runId, report) {
const hasRunError = report.aborted === true || report.evals.some((e) => e.error !== undefined);
const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
entries: [],
completed: !hasRunError,
failed: hasRunError
})
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex complete run failed: ${res.status} ${text}`);
}
}
async startEntry(runId, meta) {
const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
entries: [
{
evalName: meta.evalName,
evalType: meta.evalType ?? "capability",
description: meta.description ?? "",
tags: meta.tags ?? []
}
]
})
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex start entry failed: ${res.status} ${text}`);
}
const data = await res.json();
const entryId = data.entries?.[0]?.id ?? data.id;
if (!entryId) {
throw new Error("Vortex start entry returned no entry id");
}
return entryId;
}
async appendTurnResults(runId, entryId, turn) {
const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}/results`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ results: turnToResultRows(turn) })
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex append results failed: ${res.status} ${text}`);
}
}
async appendOutcomeResults(runId, entryId, assertions) {
if (assertions.length === 0)
return;
const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}/results`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ results: outcomeToResultRows(assertions) })
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex append results failed: ${res.status} ${text}`);
}
}
async finalizeEntry(runId, entryId, verdict) {
const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
passed: verdict.passed,
...verdict.durationMs !== undefined ? { durationMs: verdict.durationMs } : {},
...verdict.error !== undefined ? { error: verdict.error } : {}
})
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex finalize entry failed: ${res.status} ${text}`);
}
}
async markRunComplete(runId, opts = {}) {
const res = await this.fetch(`/v1/evals/runs/${runId}/complete`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ...opts.failed !== undefined ? { failed: opts.failed } : {} })
});
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex complete run failed: ${res.status} ${text}`);
}
}
async loadRunResult(runId) {
const data = await this.fetchJson(`/v1/evals/runs/${runId}`);
return vortexRunToReport(data);
}
async getLatestRun() {
const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, { limit: "1" });
if (!data.runs.length)
return null;
const latest = data.runs[0];
return this.loadRunResult(latest.id);
}
async listRunSummaries(opts) {
const runs = await this.fetchRunsWithEntries(opts?.limit ?? 50, opts?.since);
return runs.map(vortexRunToSummary);
}
async listEvalReportsByName(evalName, opts) {
const limit = opts?.limit ?? 20;
const runs = await this.fetchRunsWithEntries(50, opts?.since);
const entries = [];
for (const run of runs) {
if (entries.length >= limit)
break;
const entry = run.entries.find((e) => e.evalName === evalName);
if (entry) {
entries.push({
runId: run.id,
timestamp: run.createdAt,
report: vortexEntryToEvalReport(entry),
totalEvalsInRun: run.entries.length
});
}
}
return entries;
}
async listEvalReportsBulk(opts) {
const per = opts?.perEval ?? 20;
const runs = await this.fetchRunsWithEntries(50, opts?.since);
const grouped = {};
for (const run of runs) {
for (const entry of run.entries) {
if (!grouped[entry.evalName])
grouped[entry.evalName] = [];
if (grouped[entry.evalName].length < per) {
grouped[entry.evalName].push({
runId: run.id,
timestamp: run.createdAt,
report: vortexEntryToEvalReport(entry),
totalEvalsInRun: run.entries.length
});
}
}
}
return grouped;
}
async* watchRun(signal, options = {}) {
const POLL_INTERVAL = 1500;
const POLL_TIMEOUT = 3600000;
const start = Date.now();
const hasExpectedRun = !!options.runId || !!options.workflowId;
let baselineRunId = null;
if (!hasExpectedRun) {
try {
const initial = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
limit: "1"
});
baselineRunId = initial.runs[0]?.id ?? null;
} catch {}
}
const evalMeta = await this.loadEvalMeta();
const startedEvals = /* @__PURE__ */ new Set;
const seenTurns = /* @__PURE__ */ new Set;
const startedTurns = /* @__PURE__ */ new Set;
const finalizedEvals = /* @__PURE__ */ new Set;
while (!signal?.aborted && Date.now() - start < POLL_TIMEOUT) {
try {
const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
limit: hasExpectedRun ? "20" : "1"
});
let latest = null;
let raw = null;
for (const candidate of data.runs) {
if (options.runId && candidate.id !== options.runId)
continue;
if (options.workflowId && candidate.metadata?.workflowId !== options.workflowId) {
const candidateRaw = await this.fetchJson(`/v1/evals/runs/${candidate.id}`);
if (!runMatchesWatchOptions(candidateRaw, options))
continue;
raw = candidateRaw;
}
latest = candidate;
break;
}
if (!latest || !hasExpectedRun && latest.id === baselineRunId) {
await new Promise((r) => setTimeout(r, POLL_INTERVAL));
continue;
}
raw ??= await this.fetchJson(`/v1/evals/runs/${latest.id}`);
const terminal = raw.status === "completed" || raw.status === "failed";
for (let i = 0;i < raw.entries.length; i++) {
const entry = raw.entries[i];
const evalReport = vortexEntryToEvalReport(entry);
const meta = evalMeta.get(entry.evalName);
const totalTurns = meta?.totalTurns ?? evalReport.turns.length;
if (!startedEvals.has(entry.evalName)) {
startedEvals.add(entry.evalName);
yield { type: "eval_start", evalName: entry.evalName, index: i, totalTurns };
}
for (const turn of evalReport.turns) {
const turnKey = `${entry.evalName}::${turn.turnIndex}`;
if (seenTurns.has(turnKey))
continue;
seenTurns.add(turnKey);
yield {
type: "turn_complete",
evalName: entry.evalName,
evalIndex: i,
turnIndex: turn.turnIndex,
totalTurns,
turnReport: turn
};
}
if (entry.passed != null && !finalizedEvals.has(entry.evalName)) {
finalizedEvals.add(entry.evalName);
yield { type: "eval_complete", evalName: entry.evalName, index: i, report: evalReport };
} else if (entry.passed == null && !terminal) {
const nextTurnIndex = evalReport.turns.length;
const startKey = `${entry.evalName}::${nextTurnIndex}`;
if (nextTurnIndex < totalTurns && !startedTurns.has(startKey)) {
startedTurns.add(startKey);
yield {
type: "turn_start",
evalName: entry.evalName,
evalIndex: i,
turnIndex: nextTurnIndex,
totalTurns,
userMessage: meta?.userMessages[nextTurnIndex] ?? ""
};
}
}
}
if (terminal) {
const finalReport = vortexRunToReport(raw);
for (let i = 0;i < finalReport.evals.length; i++) {
const evalReport = finalReport.evals[i];
if (finalizedEvals.has(evalReport.name))
continue;
finalizedEvals.add(evalReport.name);
yield { type: "eval_complete", evalName: evalReport.name, index: i, report: evalReport };
}
yield {
type: "suite_complete",
report: finalReport,
error: raw.status === "failed" ? "Eval run failed" : undefined
};
return;
}
} catch {}
await new Promise((r) => setTimeout(r, POLL_INTERVAL));
}
if (!signal?.aborted) {
yield {
type: "suite_complete",
report: {
id: "timeout",
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
evals: [],
passed: 0,
failed: 0,
total: 0,
duration: 0
},
error: "Eval run timed out"
};
}
}
async getRunnerState() {
try {
const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
status: "running",
limit: "1"
});
const active = data.runs[0] ?? null;
return { running: !!active, runId: active?.id ?? null };
} catch {
return { running: false, runId: null };
}
}
async getEvalDefinitions() {
if (!this._loadEvalDefinitions)
return [];
return this._loadEvalDefinitions();
}
async loadEvalMeta() {
try {
const defs = await this.getEvalDefinitions();
return new Map(defs.map((d) => [
d.name,
{ totalTurns: d.conversation.length, userMessages: d.conversation.map((t) => t.user ?? "") }
]));
} catch {
return /* @__PURE__ */ new Map;
}
}
async fetchRunsWithEntries(limit, since) {
const data = await this.fetchJson(`/v1/evals/bot/${this.botId}/runs`, {
limit: String(limit)
});
let runs = data.runs;
if (since) {
runs = runs.filter((r) => new Date(r.createdAt).getTime() >= since);
}
return mapWithConcurrency(runs, 8, (r) => this.fetchJson(`/v1/evals/runs/${r.id}`));
}
async fetch(path, init) {
const target = new URL(`${this.url}${path}`);
const headers = new Headers(init?.headers);
if (this.token)
headers.set("Authorization", `Bearer ${this.token}`);
return globalThis.fetch(target, { ...init, headers });
}
async fetchJson(path, params) {
const target = new URL(`${this.url}${path}`);
if (params) {
for (const [k, v] of Object.entries(params))
target.searchParams.set(k, v);
}
const headers = new Headers;
if (this.token)
headers.set("Authorization", `Bearer ${this.token}`);
const res = await globalThis.fetch(target, { headers });
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex ${res.status}: ${text}`);
}
return res.json();
}
};
// src/server/utils/sse.ts
var DEFAULT_RETRY_MS = 1000;
var DEFAULT_KEEPALIVE_MS = 30000;
class SSEHub {
clients = new Set;
get size() {
return this.clients.size;
}
subscribe(client) {
this.clients.add(client);
const unsubscribe = () => {
this.clients.delete(client);
};
client.onClose(unsubscribe);
return unsubscribe;
}
broadcast(event, data) {
for (const client of this.clients) {
client.send(event, data);
}
}
}
function createSSEHeaders(req, extraHeaders = {}) {
return {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
Connection: "keep-alive",
...getCorsHeaders(req),
"X-Accel-Buffering": "no",
"Transfer-Encoding": "chunked",
...extraHeaders
};
}
function createSSEStream(req, options) {
const headers = createSSEHeaders(req, options.headers);
if (req.method === "OPTIONS") {
return new Response(null, { headers, status: 204 });
}
let client = null;
const stream = new ReadableStream({
start(controller) {
client = new SSEStreamClient(controller);
const retryMs = options.retryMs === undefined ? DEFAULT_RETRY_MS : options.retryMs;
if (retryMs !== false) {
client.write(`retry: ${retryMs}
`);
}
let keepAliveTimer = null;
if (options.keepAlive !== false) {
const keepAlive = normalizeKeepAlive(options.keepAlive);
keepAliveTimer = setInterval(() => {
client?.send(keepAlive.event, keepAlive.data);
}, keepAlive.intervalMs);
}
let connectCleanup = null;
client.onClose(() => {
if (keepAliveTimer) {
clearInterval(keepAliveTimer);
keepAliveTimer = null;
}
if (connectCleanup) {
connectCleanup();
connectCleanup = null;
}
});
let returnedCleanup;
try {
returnedCleanup = options.onConnect(client);
} catch (error) {
client.error(error);
return;
}
if (typeof returnedCleanup === "function") {
if (client.closed) {
returnedCleanup();
} else {
connectCleanup = returnedCleanup;
}
}
},
cancel() {
client?.close();
}
});
return new Response(stream, { headers });
}
function normalizeKeepAlive(keepAlive) {
if (!keepAlive || keepAlive === true) {
return { event: "keepalive", data: {}, intervalMs: DEFAULT_KEEPALIVE_MS };
}
return {
event: keepAlive.event ?? "keepalive",
data: keepAlive.data ?? {},
intervalMs: keepAlive.intervalMs ?? DEFAULT_KEEPALIVE_MS
};
}
function formatSSEEvent(options) {
const lines = [];
if (options.id !== undefined) {
lines.push(`id: ${options.id}`);
}
if (options.event) {
lines.push(`event: ${options.event}`);
}
if (options.data !== undefined) {
const data = typeof options.data === "string" ? options.data : JSON.stringify(options.data);
for (const line of data.split(/\r?\n/)) {
lines.push(`data: ${line}`);
}
}
return `${lines.join(`
`)}
`;
}
class SSEStreamClient {
controller;
encoder = new TextEncoder;
closeCallbacks = new Set;
isClosed = false;
constructor(controller) {
this.controller = controller;
}
get closed() {
return this.isClosed;
}
send(event, data, options = {}) {
return this.write(formatSSEEvent({ event, data, id: options.id }));
}
sendData(data, options = {}) {
return this.write(formatSSEEvent({ data, id: options.id }));
}
write(chunk) {
if (this.isClosed)
return false;
try {
this.controller.enqueue(this.encoder.encode(chunk));
return true;
} catch {
this.dispose();
return false;
}
}
close() {
if (this.isClosed)
return;
this.dispose();
try {
this.controller.close();
} catch {}
}
error(error) {
if (this.isClosed)
return;
this.dispose();
try {
this.controller.error(error);
} catch {}
}
onClose(callback) {
if (this.isClosed) {
callback();
return () => {};
}
this.closeCallbacks.add(callback);
return () => {
this.closeCallbacks.delete(callback);
};
}
dispose() {
if (this.isClosed)
return;
this.isClosed = true;
for (const callback of this.closeCallbacks) {
try {
callback();
} catch {}
}
this.closeCallbacks.clear();
}
}
// src/server/handlers/evals.ts
var logger2 = createCliLogger({ tag: "evals" });
var runState = {
dev: { abort: null, runId: null },
prod: { abort: null, runId: null }
};
function getActiveRunId() {
return runState.dev.runId;
}
var evalFilterSchema = ne.object({
names: ne.array(ne.string()).optional(),
tags: ne.array(ne.string()).optional(),
type: ne.enum(["capability", "regression"]).optional()
});
var runEvalsRequestSchema = ne.object({
filter: evalFilterSchema.optional()
}).passthrough();
var SINCE_WINDOW_CAP = 1e4;
function parseLimitSince(url, defaultLimit = 50) {
const limitParam = url.searchParams.get("limit");
const explicitLimit = limitParam !== null;
const limit = explicitLimit ? parseInt(limitParam, 10) : defaultLimit;
const since = url.searchParams.get("since");
const sinceTs = since ? new Date(since).getTime() : null;
const useSince = sinceTs !== null && !Number.isNaN(sinceTs);
const effectiveLimit = useSince && !explicitLimit ? SINCE_WINDOW_CAP : limit;
return {
limit: effectiveLimit,
explicitLimit,
since: useSince ? sinceTs : undefined,
invalid: explicitLimit && Number.isNaN(limit)
};
}
async function handleListEvals(store, req) {
try {
const url = new URL(req.url);
const filterParam = url.searchParams.get("filter");
let filter;
if (filterParam) {
try {
filter = evalFilterSchema.parse(JSON.parse(filterParam));
} catch {
return errorResponse("Invalid filter parameter", "filter must be a valid EvalFilter JSON object", 400);
}
}
const evals = await store.listEvals(filter);
return successResponse(evals);
} catch (err) {
return errorResponse("Failed to load evals", err.message, 500);
}
}
async function handleGetEval(store, evalName) {
try {
const def = await store.getEval(evalName);
if (!def) {
return errorResponse("Eval not found", `No eval found with name "${evalName}"`, 404);
}
return successResponse(def);
} catch (err) {
return errorResponse("Failed to load eval", err.message, 500);
}
}
async function handleRunEvals(store, req, options = {}) {
const env = options.environment ?? getActiveEnvironment();
if (runState[env].runId) {
return errorResponse("Run already in progress", `Run ${runState[env].runId} is currently executing`, 409);
}
let filter;
try {
const body = runEvalsRequestSchema.parse(await req.json());
filter = body.filter;
} catch {}
if (env === "prod") {
return handleRunEvalsProd(store, req, filter, options.prodTarget);
}
return handleRunEvalsDev(store, req, filter);
}
async function handleRunEvalsProd(store, req, filter, target) {
const config = target ? undefined : getServerConfig();
const token = target?.token ?? config?.credentials.token;
const apiUrl = target?.apiUrl ?? config?.credentials.apiUrl;
const workspaceId = target?.workspaceId ?? config?.project?.agentInfo?.workspaceId ?? config?.credentials.workspaceId;
const prodBotId = target?.botId ?? config?.project?.agentInfo?.botId ?? config?.credentials.prodBotId;
const judgeModel = target?.judgeModel ?? config?.project?.config?.evals?.judgeModel;
if (!token || !apiUrl || !workspaceId || !prodBotId) {
return errorResponse("Missing credentials", "Prod bot credentials not configured", 400);
}
const client = new Uk({
token,
botId: prodBotId,
workspaceId,
apiUrl,
headers: { "x-multiple-integrations": "true" }
});
const hasFilter = filter && Object.keys(filter).length > 0;
const abort = new AbortController;
runState.prod = { runId: "pending", abort };
return createSSEStream(req, {
keepAlive: { intervalMs: 15000 },
retryMs: false,
onConnect(sseClient) {
let workflowId = null;
let settled = false;
(async () => {
try {
const { workflow } = await client.createWorkflow({
name: "builtin_eval_runner",
status: "pending",
input: {
...hasFilter ? { filter } : {},
runType: "manual",
...judgeModel ? { judgeModel } : {}
},
timeoutAt: new Date(Date.now() + 60 * 60 * 1000).toISOString()
});
workflowId = workflow.id;
runState.prod.runId = workflow.id;
for await (const event of store.watchRun(abort.signal, { workflowId: workflow.id })) {
sseClient.sendData(event);
if (event.type === "suite_complete")
settled = true;
}
} catch (err) {
settled = true;
sseClient.sendData({
type: "suite_complete",
report: {
id: runState.prod.runId ?? "error",
timestamp: new Date().toISOString(),
evals: [],
passed: 0,
failed: 0,
total: 0,
duration: 0
},
error: err instanceof Error ? err.message : String(err)
});
} finally {
if (runState.prod.abort === abort)
runState.prod = { runId: null, abort: null };
sseClient.close();
}
})();
return () => {
abort.abort();
if (workflowId && !settled) {
client.updateWorkflow({ id: workflowId, status: "cancelled" }).catch((err) => {
logger2.error("failed to cancel workflow", { error: err instanceof Error ? err.message : String(err) });
});
}
};
}
});
}
async function handleRunEvalsDev(store, req, filter) {
const config = getServerConfig();
const { token, apiUrl } = config.credentials;
const devBotId = config.project?.agentInfo?.devId ?? config.credentials.devBotId;
if (!token || !apiUrl || !devBotId) {
return errorResponse("Missing credentials", "Bot credentials not configured. Is adk dev running?", 400);
}
const abort = new AbortController;
runState.dev = { runId: "pending", abort };
return createSSEStream(req, {
keepAlive: { intervalMs: 15000 },
retryMs: false,
onConnect(client) {
const send = (event) => void client.sendData(event);
(async () => {
let runId;
let completeRunAttempted = false;
try {
runId = await store.createRun("manual");
} catch (err) {
logger2.error("createRun failed", { error: err instanceof Error ? err.message : String(err) });
runId = crypto.randomUUID().replace(/-/g, "").slice(0, 26);
}
runState.dev.runId = runId;
const completeStoredRun = async (report) => {
completeRunAttempted = true;
try {
await store.completeRun(runId, report);
} catch (err) {
logger2.error("completeRun failed", {
runId,
error: err instanceof Error ? err.message : String(err)
});
}
};
const bpClient = new Uk({ token, botId: devBotId, apiUrl });
const chatBaseUrl = apiUrl.replace(/\/+$/, "").replace("://api.", "://chat.");
const runnerConfig = {
client: bpClient,
botId: devBotId,
agentPath: config.agentPath,
devServerUrl: `http://localhost:${config.port}`,
devServerHeaders: buildDevServerHeaders(config.agentPath),
chatClient: getChatClient(),
chatBaseUrl,
runId,
onProgress: async (event) => {
if (event.type === "eval_complete") {
try {
await store.addRunResults(runId, event.report);
} catch (err) {
logger2.error("addRunResults failed", { error: err instanceof Error ? err.message : String(err) });
}
}
if (event.type === "suite_complete") {
await completeStoredRun(event.report);
}
send(event);
},
signal: abort.signal,
evalOptions: {
idleTimeout: config.project?.config?.evals?.idleTimeout,
judgeModel: config.project?.config?.evals?.judgeModel
},
onException: (error, properties) => telemetry_default.captureException(error, properties)
};
try {
const report = await runEvalSuite(runnerConfig, filter);
if (!completeRunAttempted) {
await completeStoredRun(report);
}
} catch (err) {
send({
type: "suite_complete",
report: {
id: runId,
timestamp: new Date().toISOString(),
evals: [],
passed: 0,
failed: 0,
total: 0,
duration: 0
},
error: err instanceof Error ? err.message : String(err)
});
} finally {
if (runState.dev.abort === abort)
runState.dev = { runId: null, abort: null };
client.close();
}
})();
return () => {
abort.abort();
};
}
});
}
function handleCancelRun(options = {}) {
const env = options.environment ?? getActiveEnvironment();
const state = runState[env];
if (!state.runId) {
return successResponse({ cancelled: false, runId: null });
}
const runId = state.runId;
state.abort?.abort();
return successResponse({ cancelled: true, runId });
}
async function handleListRuns(store, req) {
const url = new URL(req.url);
const { limit, invalid, since } = parseLimitSince(url);
if (invalid)
return errorResponse("Invalid limit parameter", "limit must be numeric", 400);
try {
const summaries = await store.listRunSummaries({ limit, since });
return successResponse(summaries);
} catch (err) {
return errorResponse("Failed to list runs", err.message, 500);
}
}
async function handleGetRun(store, runId) {
try {
const run = await store.loadRunResult(runId);
if (!run) {
return errorResponse("Run not found", `No run found with ID ${runId}`, 404);
}
return successResponse(run);
} catch (err) {
return errorResponse("Failed to load run", err.message, 500);
}
}
async function handleGetLatestRun(store) {
try {
const run = await store.getLatestRun();
if (!run) {
return successResponse(null);
}
return successResponse(run);
} catch (err) {
return errorResponse("Failed to load latest run", err.message, 500);
}
}
async function handleEvalRunHistory(store, evalName, req) {
const url = new URL(req.url);
const { limit, invalid, since } = parseLimitSince(url, 20);
if (invalid)
return errorResponse("Invalid limit parameter", "limit must be numeric", 400);
try {
const results = await store.listEvalReportsByName(evalName, { limit, since });
return successResponse(results);
} catch (err) {
return errorResponse("Failed to load eval run history", err.message, 500);
}
}
async function handleEvalRunHistoryBulk(store, req) {
const url = new URL(req.url);
const perParam = url.searchParams.get("per");
const per = perParam !== null ? parseInt(perParam, 10) : 20;
if (perParam !== null && Number.isNaN(per)) {
return errorResponse("Invalid per parameter", "per must be numeric", 400);
}
const since = url.searchParams.get("since");
const sinceTs = since ? new Date(since).getTime() : null;
const useSince = sinceTs !== null && !Number.isNaN(sinceTs);
try {
const grouped = await store.listEvalReportsBulk({ perEval: per, since: useSince ? sinceTs : undefined });
return successResponse(grouped);
} catch (err) {
return errorResponse("Failed to load eval run history bulk", err.message, 500);
}
}
async function handleEvalStatus(store) {
const state = await store.getRunnerState();
return successResponse(state);
}
// src/server/eval-store-factory.ts
function resolveEvalStore(target) {
if (target) {
const store2 = createVortexStore(target.apiUrl.replace(/\/+$/, ""), target);
if (!store2) {
throw new Error("Missing credentials for production evals (botId, token, apiUrl, or workspaceId)");
}
return store2;
}
const isProd = getActiveEnvironment() === "prod";
if (!isProd) {
return createLocalStore();
}
const config = getServerConfig();
const apiUrl = config.credentials.apiUrl;
if (!apiUrl) {
throw new Error("BP_API_URL is required for production evals");
}
const store = createVortexStore(apiUrl.replace(/\/+$/, ""));
if (!store) {
throw new Error("Missing credentials for production evals (botId, token, apiUrl, or workspaceId)");
}
return store;
}
function createLocalStore() {
const config = getServerConfig();
return getLocalEvalStore(config.agentPath, getActiveRunId);
}
function createVortexStore(vortexUrl, target) {
const config = target ? undefined : getServerConfig();
const token = target?.token ?? config?.credentials.token;
const apiUrl = target?.apiUrl ?? config?.credentials.apiUrl;
const workspaceId = target?.workspaceId ?? config?.credentials.workspaceId;
const botId = target?.botId ?? config?.project?.agentInfo?.botId ?? config?.credentials.prodBotId;
if (!botId || !token || !apiUrl || !workspaceId) {
return null;
}
const manifest = createManifestLoader(token, apiUrl, workspaceId, botId);
return new VortexEvalStore({
url: vortexUrl,
botId,
workspaceId,
token,
evalManifestId: () => manifest.fileId,
loadEvalDefinitions: manifest.load
});
}
function createManifestLoader(token, apiUrl, workspaceId, botId) {
let fileId;
const load = async () => {
const client = new Uk({
token,
apiUrl,
workspaceId,
botId,
headers: { "x-multiple-integrations": "true" }
});
const { files } = await client.listFiles({ tags: EVAL_MANIFEST_TAGS });
const file = files[0];
if (!file?.url)
return [];
fileId = file.id;
const res = await fetch(file.url);
if (!res.ok)
throw new Error(`Failed to download eval manifest: ${res.status} ${res.statusText}`);
const manifest = await res.json();
if (manifest.schemaVersion !== EVAL_MANIFEST_SCHEMA_VERSION) {
throw new Error(`Eval manifest schema version ${manifest.schemaVersion} is not supported (expected ${EVAL_MANIFEST_SCHEMA_VERSION}). Redeploy the bot to update the manifest.`);
}
return manifest.evals;
};
return {
get fileId() {
return fileId;
},
load
};
}
// src/server/routes/evals.ts
function routeEvalRequest(pathname, req, store, options = {}) {
if (pathname === "/api/evals/runs/by-eval") {
return handleEvalRunHistoryBulk(store, req);
}
if (pathname.startsWith("/api/evals/runs/by-eval/")) {
const evalName = decodeURIComponent(pathname.replace("/api/evals/runs/by-eval/", ""));
if (evalName && !evalName.includes("/")) {
return handleEvalRunHistory(store, evalName, req);
}
}
if (pathname.startsWith("/api/evals/detail/")) {
const evalName = decodeURIComponent(pathname.replace("/api/evals/detail/", ""));
if (evalName && !evalName.includes("/")) {
return handleGetEval(store, evalName);
}
}
if (pathname.startsWith("/api/evals/runs/") && pathname !== "/api/evals/runs/latest") {
const runId = pathname.replace("/api/evals/runs/", "");
if (runId && !runId.includes("/")) {
return handleGetRun(store, runId);
}
}
switch (pathname) {
case "/api/evals":
return handleListEvals(store, req);
case "/api/evals/run": {
const methodError = validateMethod(req, "POST");
if (methodError)
return Promise.resolve(methodError);
return handleRunEvals(store, req, options);
}
case "/api/evals/cancel": {
const methodError = validateMethod(req, "POST");
if (methodError)
return Promise.resolve(methodError);
return Promise.resolve(handleCancelRun(options));
}
case "/api/evals/runs":
return handleListRuns(store, req);
case "/api/evals/runs/latest":
return handleGetLatestRun(store);
case "/api/evals/status":
return handleEvalStatus(store);
default:
return Promise.resolve(errorResponse("Not Found", "Not Found", 404));
}
}
// src/server/prod-bot-api-handler.ts
var AGENT_NOT_DEPLOYED_MESSAGE2 = "Agent has not been deployed yet. Deploy with `adk deploy` to publish the production target.";
function getVortexUrl(prodBotSelection) {
const apiUrl = prodBotSelection.apiUrl;
if (!apiUrl)
return null;
return apiUrl.replace(/\/+$/, "");
}
function vortexHeaders(prodBotSelection) {
const headers = new Headers;
headers.set("Authorization", `Bearer ${prodBotSelection.token}`);
return headers;
}
async function fetchVortex(vortexPath, prodBotSelection, query) {
const vortexUrl = getVortexUrl(prodBotSelection);
if (!vortexUrl)
throw new Error("BP_API_URL is required for production traces and eval history.");
const target = new URL(`${vortexUrl}${vortexPath}`);
if (query)
target.search = query.toString();
const res = await fetch(target, { headers: vortexHeaders(prodBotSelection) });
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(`Vortex ${res.status}: ${text}`);
}
return res.json();
}
function vortexSpanToUi(s) {
const tier = s.data["adk.tier"] ?? "verbose";
const { "adk.tier": _, importance: __, ...data } = s.data;
const status = normalizeVortexSpanStatus(s);
return {
...s,
status,
label: s.name,
tier,
data,
resource: { environment: "production", versions: {} }
};
}
function normalizeVortexSpanStatus(s) {
if (s.status !== "running")
return s.status;
const data = s.data;
const autonomousStatus = data["autonomous.status"];
if (autonomousStatus === "generation_error" || autonomousStatus === "execution_error" || autonomousStatus === "invalid_code_error" || autonomousStatus === "exit_error" || autonomousStatus === "aborted") {
return "error";
}
if (autonomousStatus === "thinking_requested" || autonomousStatus === "callback_requested" || autonomousStatus === "exit_success") {
return "ok";
}
if (hasEndedTiming(s))
return "ok";
return "running";
}
function hasEndedTiming(s) {
const { startedAt, endedAt } = s.timing;
if (typeof endedAt === "number" && Number.isFinite(endedAt) && endedAt > startedAt)
return true;
return false;
}
function normalizeVortexTimestampParam(value) {
const numericValue = Number(value);
const date = Number.isFinite(numericValue) ? new Date(numericValue) : new Date(value);
if (Number.isNaN(date.getTime()))
return value;
return date.toISOString();
}
async function handleProdTracesQuery(url, prodBotSelection) {
if (!getVortexUrl(prodBotSelection))
return successResponse([]);
try {
const query = new URLSearchParams;
const count = url.searchParams.get("count");
if (count)
query.set("limit", count);
const fromTimestamp = url.searchParams.get("fromTimestamp") ?? url.searchParams.get("startTs");
if (fromTimestamp)
query.set("fromTimestamp", normalizeVortexTimestampParam(fromTimestamp));
const toTimestamp = url.searchParams.get("toTimestamp") ?? url.searchParams.get("endTs");
if (toTimestamp)
query.set("toTimestamp", normalizeVortexTimestampParam(toTimestamp));
const attributeName = url.searchParams.get("attributeName");
const attributeValue = url.searchParams.get("attributeValue");
if (attributeName && attributeValue) {
query.set(attributeName, attributeValue);
}
const data = await fetchVortex(`/v1/traces/bot/${prodBotSelection.botId}/spans/adk`, prodBotSelection, query);
return successResponse(data.spans.map(vortexSpanToUi));
} catch (err) {
return errorResponse("Failed to fetch traces", err instanceof Error ? err.message : String(err), 502);
}
}
async function handleProdTraceById(url, prodBotSelection) {
const traceId = url.searchParams.get("traceId");
if (!traceId)
return errorResponse("Missing traceId", "traceId query param required", 400);
if (!getVortexUrl(prodBotSelection))
return successResponse([]);
try {
const data = await fetchVortex(`/v1/traces/bot/${prodBotSelection.botId}/${traceId}/adk`, prodBotSelection);
return successResponse(data.spans.map(vortexSpanToUi));
} catch (err) {
return errorResponse("Failed to fetch trace", err instanceof Error ? err.message : String(err), 502);
}
}
async function handleProdRecentTraces(url, prodBotSelection) {
if (!getVortexUrl(prodBotSelection))
return successResponse([]);
try {
const query = new URLSearchParams;
const limit = url.searchParams.get("limit");
if (limit)
query.set("limit", limit);
const data = await fetchVortex(`/v1/traces/bot/${prodBotSelection.botId}/spans/adk`, prodBotSelection, query);
return successResponse(data.spans.map(vortexSpanToUi));
} catch (err) {
return errorResponse("Failed to fetch traces", err instanceof Error ? err.message : String(err), 502);
}
}
async function handleProdTodayTraces(prodBotSelection) {
if (!getVortexUrl(prodBotSelection))
return successResponse([]);
try {
const query = new URLSearchParams;
const today = new Date;
today.setHours(0, 0, 0, 0);
query.set("fromTimestamp", today.toISOString());
const data = await fetchVortex(`/v1/traces/bot/${prodBotSelection.botId}/spans/adk`, prodBotSelection, query);
return successResponse(data.spans.map(vortexSpanToUi));
} catch (err) {
return errorResponse("Failed to fetch traces", err instanceof Error ? err.message : String(err), 502);
}
}
function handleProdEvalRequest(pathname, req, prodBotSelection) {
const store = resolveEvalStore({
token: prodBotSelection.token,
apiUrl: prodBotSelection.apiUrl,
workspaceId: prodBotSelection.workspaceId,
botId: prodBotSelection.botId
});
return routeEvalRequest(pathname, req, store, {
environment: "prod",
prodTarget: {
token: prodBotSelection.token,
apiUrl: prodBotSelection.apiUrl,
workspaceId: prodBotSelection.workspaceId,
botId: prodBotSelection.botId
}
});
}
async function handleProdBotApiRequest(url, req, prodBotSelection) {
const pathname = url.pathname;
if ((pathname.startsWith("/api/traces") || pathname.startsWith("/api/evals")) && !isProductionObservabilityEnabled()) {
return productionObservabilityDisabledResponse();
}
if (pathname === "/api/traces/query")
return handleProdTracesQuery(url, prodBotSelection);
if (pathname === "/api/traces/trace")
return handleProdTraceById(url, prodBotSelection);
if (pathname === "/api/traces/recent")
return handleProdRecentTraces(url, prodBotSelection);
if (pathname === "/api/traces/today")
return handleProdTodayTraces(prodBotSelection);
if (pathname === "/api/traces/stream") {
return errorResponse("Not available", "Live trace streaming is not available for production bots.", 404);
}
if (pathname.startsWith("/api/evals"))
return handleProdEvalRequest(pathname, req, prodBotSelection);
switch (pathname) {
case "/api/config":
return successResponse({
credentials: {
hasToken: true,
token: prodBotSelection.token,
apiUrl: prodBotSelection.apiUrl,
workspaceId: prodBotSelection.workspaceId,
prodBotId: prodBotSelection.botId
},
agentPath: prodBotSelection.agentPath,
identity: {},
devBot: { running: false, port: null, url: null, botId: null }
});
case "/api/agent":
try {
const agentDefinition = await prodAgentMetadataService.getAgentDefinition({
token: prodBotSelection.token,
apiUrl: prodBotSelection.apiUrl,
workspaceId: prodBotSelection.workspaceId,
botId: prodBotSelection.botId,
botName: prodBotSelection.botName,
agentPath: prodBotSelection.agentPath
});
return successResponse(agentDefinition);
} catch (error) {
return errorResponse("Agent not deployed", error instanceof Error ? error.message : AGENT_NOT_DEPLOYED_MESSAGE2, 404);
}
case "/api/agent-map/snapshot":
try {
const snapshot = await prodAgentMapSnapshotService.getSnapshot({
token: prodBotSelection.token,
apiUrl: prodBotSelection.apiUrl,
workspaceId: prodBotSelection.workspaceId,
botId: prodBotSelection.botId
});
return successResponse(snapshot);
} catch (error) {
return errorResponse(error instanceof AgentMapSnapshotNotPublishedError ? "Agent Map metadata not published" : "Snapshot failed", error instanceof Error ? error.message : String(error), error instanceof AgentMapSnapshotNotPublishedError ? 404 : 500);
}
case "/api/environment":
if (req.method === "POST") {
return errorResponse("Not allowed", "Cloud Dev Console mode already targets Botpress Cloud.", 400);
}
return successResponse({ environment: "prod" });
case "/api/config/variables": {
const env = url.searchParams.get("env");
if (env && env !== "prod") {
return errorResponse("Invalid environment", "Cloud Dev Console mode only supports the prod target.", 400);
}
const target = {
token: prodBotSelection.token,
apiUrl: prodBotSelection.apiUrl,
workspaceId: prodBotSelection.workspaceId,
botId: prodBotSelection.botId
};
if (req.method === "GET") {
return handleGetProdConfigVariables(target);
}
if (req.method === "PUT") {
return handlePutProdConfigVariables(req, target);
}
if (req.method === "PATCH") {
return errorResponse("schema_mutation_requires_deploy", "Prod schema changes must go through `adk deploy`.", 409);
}
return errorResponse("Method not allowed", "Only GET and PUT are supported in Cloud Dev Console mode.", 405);
}
case "/api/config/variables/diff":
return errorResponse("Not available", "Cloud Dev Console mode does not have local dev configuration schema to diff against.", 404);
case "/api/health":
return successResponse({ status: "ok" });
case "/api/feature-flags":
return successResponse({ flags: {} });
default:
return errorResponse("Not available", "This endpoint is not available in Cloud Dev Console mode.", 404);
}
}
// src/server/handlers/feature-flags.ts
var UI_FLAGS = ["enable_agent_0", "enable_guided_setup"];
async function handleFeatureFlags() {
const flags = {};
for (const name of UI_FLAGS) {
flags[name] = isFeatureEnabled(name);
}
return successResponse({ flags });
}
export { getCorsHeaders, handleCorsPreflightResponse, withRequestTiming, startEventLoopLagMonitor, jsonResponse, errorResponse, successResponse, prodAgentMetadataService, getLocalProdMetadataTarget, getServerConfig, setServerConfig, getServerStartTime, setServerStartTime, getDevCommandStatus, setDevCommandStatus, getLatestWorkerStats, updateWorkerStats, getAgent0RuntimeClient, setAgent0RuntimeClient, getActiveEnvironment, setActiveEnvironment, getDevBotRuntimeState, setDevBotRuntimeState, resetDevBotRuntimeState, emitSecretsValuesChanged, onSecretsValuesChanged, emitProjectReloaded, onProjectReloaded, parseEnv, validateCredentials, validateBotId, validateMethod, validateProjectAndCredentials, getTargetBotId, getScopedServerCredentials, handleGetConfigVariables, handlePutConfigVariables, handlePatchConfigSchema, handleGetConfigSchemaDiff, isProductionObservabilityEnabled, productionObservabilityDisabledResponse, getLocalEvalStore, SSEHub, createSSEStream, resolveEvalStore, routeEvalRequest, handleProdBotApiRequest, handleFeatureFlags };