@agentled/cli
Version:
CLI for Agentled — manage workflows, apps, and knowledge from the command line. Zero context-window cost for AI agents.
283 lines • 12.4 kB
JavaScript
/**
* Declarative test runner — runs assertions against captured fixture outputs
* (zero credits) or live API calls (`--live`, costs credits for AI/app steps).
*
* Test file format (tests/<workflowId>.test.json):
* {
* "workflowId": "wf_abc",
* "fixtureExecutionId": "exec_xyz",
* "steps": [
* {
* "stepId": "score-lead",
* "type": "aiAction",
* "fixtureSource": "exec_xyz/score-lead",
* "assertions": [
* { "verb": "hasKey", "path": "score" },
* { "verb": "isType", "path": "score", "expect": "number" },
* { "verb": "inRange", "path": "score", "min": 0, "max": 100 },
* { "verb": "matches", "path": "decision", "pattern": "^(HOT|WARM|COLD)$" }
* ]
* }
* ]
* }
*
* Assertion verbs: hasKey, isType, equals, matches, inRange, hasLength, contains
*/
import { existsSync, readFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
// ---------------------------------------------------------------------------
// Path accessor
// ---------------------------------------------------------------------------
export function getPath(obj, dotPath) {
if (!dotPath || dotPath === '.')
return obj;
const parts = dotPath.split('.');
let cur = obj;
for (const part of parts) {
if (cur === null || cur === undefined)
return undefined;
if (Array.isArray(cur)) {
const idx = parseInt(part, 10);
cur = isNaN(idx) ? undefined : cur[idx];
}
else if (typeof cur === 'object') {
cur = cur[part];
}
else {
return undefined;
}
}
return cur;
}
// ---------------------------------------------------------------------------
// Assertion runner
// ---------------------------------------------------------------------------
export function runAssertion(output, assertion) {
const value = getPath(output, assertion.path);
const { verb } = assertion;
switch (verb) {
case 'hasKey': {
const pass = value !== undefined && value !== null;
return { pass, reason: pass ? '' : `Expected key "${assertion.path}" present and non-null, got: ${JSON.stringify(value)}` };
}
case 'isType': {
const expect = assertion.expect;
const actual = Array.isArray(value) ? 'array' : typeof value;
return { pass: actual === expect, reason: `Expected "${assertion.path}" type "${expect}", got "${actual}"` };
}
case 'equals': {
const pass = JSON.stringify(value) === JSON.stringify(assertion.expect);
return { pass, reason: pass ? '' : `Expected "${assertion.path}" to equal ${JSON.stringify(assertion.expect)}, got ${JSON.stringify(value)}` };
}
case 'matches': {
if (typeof value !== 'string')
return { pass: false, reason: `Expected "${assertion.path}" string for regex, got ${typeof value}` };
const re = new RegExp(assertion.pattern ?? '');
const pass = re.test(value);
return { pass, reason: pass ? '' : `Expected "${assertion.path}" to match /${assertion.pattern}/, got "${value}"` };
}
case 'inRange': {
if (typeof value !== 'number')
return { pass: false, reason: `Expected "${assertion.path}" number, got ${typeof value}` };
const min = assertion.min ?? -Infinity;
const max = assertion.max ?? Infinity;
const pass = value >= min && value <= max;
return { pass, reason: pass ? '' : `Expected "${assertion.path}" between ${min} and ${max}, got ${value}` };
}
case 'hasLength': {
if (!Array.isArray(value) && typeof value !== 'string')
return { pass: false, reason: `Expected "${assertion.path}" array/string, got ${typeof value}` };
const len = value.length;
if (assertion.expect !== undefined) {
const pass = len === Number(assertion.expect);
return { pass, reason: pass ? '' : `Expected length ${assertion.expect}, got ${len}` };
}
return { pass: len > 0, reason: `Expected non-empty, got length ${len}` };
}
case 'contains': {
if (typeof value === 'string') {
const pass = value.includes(String(assertion.expect ?? ''));
return { pass, reason: pass ? '' : `Expected "${assertion.path}" to contain "${assertion.expect}", got "${value}"` };
}
if (Array.isArray(value)) {
const pass = value.some((item) => JSON.stringify(item) === JSON.stringify(assertion.expect));
return { pass, reason: pass ? '' : `Expected array to contain ${JSON.stringify(assertion.expect)}` };
}
return { pass: false, reason: `Expected "${assertion.path}" string/array for contains` };
}
default:
return { pass: false, reason: `Unknown assertion verb: "${verb}"` };
}
}
// ---------------------------------------------------------------------------
// Fixture loader
// ---------------------------------------------------------------------------
export function loadFixture(wsDir, fixtureSource) {
const parts = fixtureSource.split('/');
if (parts.length < 2)
return null;
const [executionId, ...stepParts] = parts;
const stepId = stepParts.join('/');
const candidates = [];
if (wsDir)
candidates.push(join(wsDir, 'fixtures', 'step-outputs', executionId, `${stepId}.json`));
candidates.push(resolve('fixtures', 'step-outputs', executionId, `${stepId}.json`));
for (const candidate of candidates) {
if (existsSync(candidate)) {
try {
return JSON.parse(readFileSync(candidate, 'utf-8'));
}
catch {
return null;
}
}
}
return null;
}
// ---------------------------------------------------------------------------
// Run a step in fixture mode (zero credits)
// ---------------------------------------------------------------------------
export function runStepFixture(step, wsDir) {
const result = { stepId: step.stepId, description: step.description, passed: 0, failed: 0, skipped: 0, errors: [], mode: 'fixture' };
if (!step.fixtureSource) {
result.mode = 'skipped';
result.skipped = step.assertions.length;
result.errors.push('No fixtureSource set — run `agentled fixture capture` first.');
return result;
}
const fixture = loadFixture(wsDir, step.fixtureSource);
if (!fixture) {
result.mode = 'skipped';
result.skipped = step.assertions.length;
result.errors.push(`Fixture not found: ${step.fixtureSource}`);
return result;
}
const output = fixture.output ?? fixture;
for (const assertion of step.assertions) {
if (assertion.path === '?') {
result.skipped++;
continue;
}
const { pass, reason } = runAssertion(output, assertion);
if (pass)
result.passed++;
else {
result.failed++;
result.errors.push(reason);
}
}
return result;
}
export async function runStepLive(step, client) {
const result = { stepId: step.stepId, description: step.description, passed: 0, failed: 0, skipped: 0, errors: [], mode: 'live' };
let output;
try {
if (step.type === 'code') {
const response = await client.testCodeAction(step.codeConfig?.code ?? '', step.codeConfig?.language ?? 'javascript', step.liveInputs ?? {});
output = response.output ?? response.result ?? response;
}
else if (step.type === 'aiAction' || step.type === 'aiActionWithTools') {
if (!step.promptTemplate) {
result.mode = 'skipped';
result.skipped = step.assertions.length;
result.errors.push('Live mode requires "promptTemplate".');
return result;
}
const response = await client.testAiAction(step.promptTemplate, step.promptVariables ?? step.liveInputs ?? {}, step.promptResponseStructure ?? {});
output = response.output ?? response.result ?? response;
}
else if (step.type === 'appAction') {
if (!step.appId || !step.actionId) {
result.mode = 'skipped';
result.skipped = step.assertions.length;
result.errors.push('Live mode requires "appId" and "actionId".');
return result;
}
const response = await client.testAppAction(step.appId, step.actionId, step.appInputs ?? step.liveInputs ?? {});
output = response.output ?? response.result ?? response;
}
else {
result.mode = 'skipped';
result.skipped = step.assertions.length;
result.errors.push(`Step type "${step.type}" doesn't support live testing.`);
return result;
}
}
catch (err) {
result.failed = step.assertions.length;
result.errors.push(`API call failed: ${err instanceof Error ? err.message : String(err)}`);
return result;
}
for (const assertion of step.assertions) {
if (assertion.path === '?') {
result.skipped++;
continue;
}
const { pass, reason } = runAssertion(output, assertion);
if (pass)
result.passed++;
else {
result.failed++;
result.errors.push(reason);
}
}
return result;
}
export function makeTestSkeleton(workflowId, pipeline) {
const allSteps = (Array.isArray(pipeline.steps) ? pipeline.steps : []);
const testable = allSteps.filter((s) => ['aiAction', 'aiActionWithTools', 'appAction', 'code'].includes(String(s.type ?? '')));
return {
workflowId,
workflowName: String(pipeline.name ?? workflowId),
description: `Auto-generated test skeleton for ${pipeline.name ?? workflowId}. Edit assertions to match expected outputs.`,
fixtureExecutionId: null,
steps: testable.map((s) => {
const stepId = String(s.id ?? '');
const type = String(s.type ?? '');
const app = s.app;
const test = {
stepId,
type,
description: String(s.name ?? stepId),
fixtureSource: null,
assertions: [],
};
if (type === 'aiAction' || type === 'aiActionWithTools') {
const responseStructure = s.pipelineStepPrompt?.responseStructure;
if (responseStructure && typeof responseStructure === 'object') {
test.assertions = Object.keys(responseStructure).slice(0, 5).map((key) => ({
verb: 'hasKey',
path: key,
comment: `Step should output a "${key}" field`,
}));
}
else {
test.assertions = [{ verb: 'hasKey', path: '?', comment: 'Replace ? with an expected output field' }];
}
}
else if (type === 'appAction') {
const actionId = String(app?.actionId ?? '');
if (actionId.includes('read-list')) {
test.assertions = [
{ verb: 'hasKey', path: 'listEntries' },
{ verb: 'isType', path: 'listEntries', expect: 'array' },
];
}
else if (actionId.includes('find-email')) {
test.assertions = [
{ verb: 'hasKey', path: 'email' },
{ verb: 'matches', path: 'email', pattern: '.+@.+\\..+' },
];
}
else {
test.assertions = [{ verb: 'hasKey', path: '?', comment: 'Replace ? with an expected output field' }];
}
}
else if (type === 'code') {
test.assertions = [{ verb: 'hasKey', path: '?', comment: 'Replace ? with an expected output field' }];
}
return test;
}),
};
}
//# sourceMappingURL=test-runner.js.map