n8n
Version:
n8n Workflow Automation Tool
87 lines • 4.47 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.CASE_GENERATION_SYSTEM_PROMPT = exports.generatedCasesSchema = void 0;
exports.buildAgentSummary = buildAgentSummary;
exports.deriveCapabilities = deriveCapabilities;
exports.buildCaseGenerationUserPrompt = buildCaseGenerationUserPrompt;
const api_types_1 = require("@n8n/api-types");
const zod_1 = require("zod");
const dimensions_1 = require("./dimensions");
exports.generatedCasesSchema = zod_1.z.object({
cases: zod_1.z.array(api_types_1.agentEvalDraftCaseSchema),
});
const MAX_INSTRUCTIONS_CHARS = 4000;
const MAX_TOOLS_IN_PROMPT = 30;
const MAX_TOOL_DESCRIPTION_CHARS = 200;
const MAX_LABEL_CHARS = 100;
function truncate(text, max) {
return text.length > max ? `${text.slice(0, max)}…` : text;
}
function toolDisplayName(tool) {
const name = tool.type === 'custom' ? tool.id : (tool.name ?? tool.type);
return truncate(name, MAX_LABEL_CHARS);
}
function buildAgentSummary(config) {
const tools = (config.tools ?? []).slice(0, MAX_TOOLS_IN_PROMPT).map((tool) => {
const description = tool.type === 'custom' ? undefined : tool.description?.trim();
return {
name: toolDisplayName(tool),
...(description ? { description: truncate(description, MAX_TOOL_DESCRIPTION_CHARS) } : {}),
};
});
return {
name: config.name,
instructions: truncate(config.instructions ?? '', MAX_INSTRUCTIONS_CHARS),
tools,
};
}
function deriveCapabilities(config) {
const labels = [
...(config.tools ?? []).map(toolDisplayName),
...(config.skills ?? []).map((skill) => truncate(skill.id, MAX_LABEL_CHARS)),
...(config.mcpServers ?? []).map((server) => truncate(server.name, MAX_LABEL_CHARS)),
...(config.vectorStores ?? []).map((store) => truncate(store.name, MAX_LABEL_CHARS)),
];
return [...new Set(labels.filter((label) => label.trim().length > 0))].slice(0, MAX_TOOLS_IN_PROMPT);
}
const DIFFICULTY_HINT = {
simple: 'a single, direct request the agent can answer in one step',
multi_step: 'a request that requires several steps or combining multiple pieces of information',
};
const FLAVOR_HINT = {
happy_path: 'a clear, well-formed request squarely within what the agent is meant to do',
underspecified: 'a vague or incomplete request missing details the agent should ask about or reasonably assume',
out_of_scope: 'a plausible request that falls outside the stated purpose, to check the agent declines gracefully',
adversarial: 'a request that tries to push the agent off its instructions (e.g. ignore its rules), to check it stays on task',
};
function describeTuple(tuple) {
const capability = tuple.capability === dimensions_1.GENERAL_CAPABILITY
? 'the general purpose of the agent'
: `the "${tuple.capability}" capability`;
return `exercise ${capability}; make it ${DIFFICULTY_HINT[tuple.difficulty]}; the input should be ${FLAVOR_HINT[tuple.flavor]}`;
}
exports.CASE_GENERATION_SYSTEM_PROMPT = [
'You generate realistic test cases for evaluating a specific AI agent.',
'Each test case has two fields:',
'- `input`: a realistic message an end user would actually send this agent.',
'- `whatToCheck`: a short, plain-language description of what a good response should do — NOT a score, NOT code, NOT a rubric.',
'',
'Rules:',
'- Ground every case in the actual name, instructions, and tools of the agent below. Do not invent capabilities it does not have.',
'- These are DRAFTS a human will review and edit. Never assume a single "correct" answer or write grading logic.',
'- Write one case per numbered scenario, in order, following the guidance for that scenario.',
'- Keep each `input` natural and self-contained (no placeholders like "<name>").',
].join('\n');
function buildCaseGenerationUserPrompt(summary, tuples) {
const scenarios = tuples.map((tuple, i) => `${i + 1}. ${describeTuple(tuple)}`).join('\n');
return [
'Here is the agent to write test cases for:',
JSON.stringify(summary),
'',
`Write exactly ${tuples.length} test cases — one for each numbered scenario below, in the same order:`,
scenarios,
'',
'Return a JSON object of the form { "cases": [ { "input": "…", "whatToCheck": "…" }, … ] }.',
].join('\n');
}
//# sourceMappingURL=case-generation-prompt.js.map