@agentled/cli
Version:
CLI for Agentled — manage workflows, apps, and knowledge from the command line. Zero context-window cost for AI agents.
661 lines • 26.7 kB
JavaScript
/**
* `agentled dryrun <file>` — zero-credit variable-resolution walk over a pipeline.
*
* Walks the step graph from the trigger, synthesizes mock outputs for each step
* (from responseStructure / app output schemas / fixture data), substitutes
* template variables, and reports:
*
* - Unresolved `{{steps.X.field}}` / `{{input.Y}}` / `{{currentItem.Z}}` refs
* - Always-skip / always-stop branches based on entryConditions
* - Credit estimate (sum of step.creditCost or per-action defaults)
*
* Inputs (in priority order, latest wins):
* 1. Synthetic defaults from context.executionInputConfig.fields
* 2. --input '<json>' or --input-file <path>
* 3. --mocks <path> — full {stepId: output} object overriding any synthetic
* 4. --from-execution <execId> — load each step's output from
* fixtures/step-outputs/<execId>/<stepId>.json (most realistic)
*
* Exit codes: 0 clean, 1 warnings (always-skip branches, no terminal step),
* 2 unresolved refs.
*/
import { existsSync, readdirSync, readFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { printOutput, printError } from '../utils/output.js';
import { findWorkspaceDir } from '../utils/workspace-folder.js';
const VALIDATION_EXIT_CODE = 2;
const WARNING_EXIT_CODE = 1;
// ---------------------------------------------------------------------------
// Pipeline helpers
// ---------------------------------------------------------------------------
function steps(p) {
return Array.isArray(p.steps) ? p.steps : [];
}
function findStepById(allSteps, id) {
return allSteps.find((s) => String(s.id ?? '') === id);
}
function findTrigger(allSteps) {
return allSteps.find((s) => s.type === 'trigger') ?? allSteps[0];
}
function getNextStepId(step) {
const next = step.next;
if (!next)
return undefined;
const id = next.stepId;
return typeof id === 'string' && id ? id : undefined;
}
// ---------------------------------------------------------------------------
// Template substitution
// ---------------------------------------------------------------------------
const TEMPLATE_RE = /\{\{\s*([^}]+?)\s*\}\}/g;
function getDeep(obj, path) {
let cur = obj;
for (const part of path) {
if (cur === null || cur === undefined)
return { found: false, value: undefined };
if (Array.isArray(cur)) {
const idx = parseInt(part, 10);
if (isNaN(idx))
return { found: false, value: undefined };
cur = cur[idx];
continue;
}
if (typeof cur === 'object') {
const rec = cur;
if (!(part in rec))
return { found: false, value: undefined };
cur = rec[part];
continue;
}
return { found: false, value: undefined };
}
return { found: true, value: cur };
}
function resolveExpression(expression, state) {
// Strip filter pipes (`| filter`) and fallback (`|| 'default'`) before
// resolving the leftmost path. This matches docs/TEMPLATE_SYNTAX.md —
// both `{{a || 'fb'}}` and `{{array | where:x | first}}` should resolve
// to the leftmost identifier (`a` / `array`). We still report unresolved
// refs on the left side (catches typos in step IDs) without false-failing
// on legitimate fallbacks/filters.
const leftmost = expression.split(/\s*\|\|?\s*/)[0].trim();
if (!leftmost)
return { resolved: true, value: undefined };
const parts = leftmost.split('.');
const root = parts[0];
const rest = parts.slice(1);
if (root === 'input') {
return resolveAndReport({ found: true, value: state.input }, rest, root);
}
if (root === 'currentItem') {
if (state.currentItem === undefined)
return { resolved: false, value: undefined, reason: 'currentItem set only inside loops' };
return resolveAndReport({ found: true, value: state.currentItem }, rest, root);
}
if (root === 'context') {
return resolveAndReport({ found: true, value: state.context }, rest, root);
}
if (root === 'steps') {
if (rest.length === 0)
return { resolved: false, value: undefined, reason: 'expression "steps" needs a step ID' };
const stepId = rest[0];
if (!(stepId in state.steps)) {
return { resolved: false, value: undefined, reason: `step "${stepId}" has no recorded output yet (forward ref or missing step)` };
}
return resolveAndReport({ found: true, value: state.steps[stepId] }, rest.slice(1), `steps.${stepId}`);
}
if (root === 'execution' || root === 'workspace') {
// Always resolvable at runtime, treat as known
return { resolved: true, value: `<${root}>` };
}
return { resolved: false, value: undefined, reason: `unknown root identifier "${root}"` };
}
function resolveAndReport(start, rest, rootLabel) {
if (rest.length === 0)
return { resolved: true, value: start.value };
const got = getDeep(start.value, rest);
if (!got.found) {
return {
resolved: false,
value: undefined,
reason: `path "${rest.join('.')}" not found on ${rootLabel} (mock output may be missing this field)`,
};
}
return { resolved: true, value: got.value };
}
function findExpressionsAndCheck(value, state, stepId, fieldPath, unresolved) {
if (value === null || value === undefined)
return;
if (typeof value === 'string') {
const matches = value.matchAll(TEMPLATE_RE);
for (const m of matches) {
const expr = m[1];
const result = resolveExpression(expr, state);
if (!result.resolved) {
// If the expression has a fallback (`||`), runtime will use the
// fallback value when the leftmost path is missing. Downgrade
// to warning — still surfaces typos but doesn't fail the dry-run.
const hasFallback = /\|\|/.test(expr);
unresolved.push({
stepId,
field: fieldPath,
expression: m[0],
reason: result.reason ?? 'unresolved',
severity: hasFallback ? 'warning' : 'error',
});
}
}
return;
}
if (Array.isArray(value)) {
value.forEach((item, idx) => findExpressionsAndCheck(item, state, stepId, `${fieldPath}[${idx}]`, unresolved));
return;
}
if (typeof value === 'object') {
for (const [k, v] of Object.entries(value)) {
findExpressionsAndCheck(v, state, stepId, fieldPath ? `${fieldPath}.${k}` : k, unresolved);
}
}
}
// ---------------------------------------------------------------------------
// Mock output synthesizer
// ---------------------------------------------------------------------------
function synthesizeMockOutput(step, appsCache) {
const type = String(step.type ?? '');
if (type === 'aiAction' || type === 'aiActionWithTools') {
const prompt = step.pipelineStepPrompt;
const responseStructure = prompt?.responseStructure;
return shapeFromResponseStructure(responseStructure);
}
if (type === 'appAction') {
const app = step.app;
if (app && appsCache) {
const schema = lookupAppOutputSchema(appsCache, String(app.id ?? ''), String(app.actionId ?? ''));
if (schema)
return shapeFromOutputSchema(schema);
}
return {};
}
if (type === 'code') {
// Without --live, we don't run the code. Empty object signals "any shape".
return {};
}
if (type === 'setVariables') {
const cfg = step.setVariablesConfig;
const vars = cfg?.variables;
const obj = {};
if (Array.isArray(vars)) {
for (const v of vars) {
const name = v?.name;
if (typeof name === 'string')
obj[name] = null;
}
}
return obj;
}
if (type === 'knowledgeSync') {
return { addedCount: 0, listKey: null };
}
return {};
}
function shapeFromResponseStructure(rs) {
if (!rs || typeof rs !== 'object')
return {};
const out = {};
for (const [k, v] of Object.entries(rs)) {
if (v && typeof v === 'object' && !Array.isArray(v)) {
out[k] = shapeFromResponseStructure(v);
continue;
}
const desc = String(v ?? '').toLowerCase();
if (desc.includes('array') || desc.includes('list'))
out[k] = [];
else if (desc.includes('number') || desc.includes('int') || desc.includes('float') || desc.includes('score'))
out[k] = 0;
else if (desc.includes('bool'))
out[k] = false;
else if (desc.includes('object'))
out[k] = {};
else
out[k] = '';
}
return out;
}
function lookupAppOutputSchema(appsCache, appId, actionId) {
const apps = (appsCache.apps ?? appsCache);
if (!Array.isArray(apps))
return null;
const app = apps.find((a) => a.id === appId);
const actions = app?.actions;
if (!Array.isArray(actions))
return null;
const action = actions.find((a) => a.id === actionId || a.actionId === actionId);
return action?.outputSchema ?? action?.output ?? null;
}
function shapeFromOutputSchema(schema) {
if (schema.type === 'object' || schema.properties) {
const props = (schema.properties ?? {});
const out = {};
for (const [k, v] of Object.entries(props)) {
out[k] = shapeFromOutputSchema(v);
}
return out;
}
if (schema.type === 'array')
return [];
if (schema.type === 'number' || schema.type === 'integer')
return 0;
if (schema.type === 'boolean')
return false;
return '';
}
// ---------------------------------------------------------------------------
// Synthetic input from executionInputConfig.fields
// ---------------------------------------------------------------------------
function synthesizeInput(pipeline) {
const ctx = pipeline.context;
const cfg = ctx?.executionInputConfig;
const fields = cfg?.fields;
if (!Array.isArray(fields))
return {};
const input = {};
for (const f of fields) {
const name = String(f.name ?? '');
if (!name)
continue;
const type = String(f.type ?? 'text').toLowerCase();
if (type.includes('number'))
input[name] = 1;
else if (type.includes('bool'))
input[name] = true;
else if (type.includes('array') || type.includes('multiple'))
input[name] = [];
else if (type.includes('url'))
input[name] = 'https://example.com';
else if (type.includes('email'))
input[name] = 'sample@example.com';
else
input[name] = `sample_${name}`;
}
return input;
}
function synthesizeContext(pipeline) {
const ctx = pipeline.context;
const inputPages = ctx?.inputPages;
if (!Array.isArray(inputPages))
return {};
const out = {};
for (const page of inputPages) {
const config = page.configuration;
const contextKey = config?.contextKey;
if (typeof contextKey !== 'string')
continue;
const fields = config?.fields;
const synth = {};
if (Array.isArray(fields)) {
for (const f of fields) {
const name = String(f.name ?? '');
if (name)
synth[name] = `<${contextKey}.${name}>`;
}
}
out[contextKey] = synth;
}
return out;
}
// ---------------------------------------------------------------------------
// Entry condition simulation
// ---------------------------------------------------------------------------
function simulateEntryConditions(step, state) {
const ec = step.entryConditions;
if (!ec)
return { run: true };
const criteria = ec.criteria;
if (!Array.isArray(criteria) || criteria.length === 0)
return { run: true };
const onFail = String(ec.onCriteriaFail ?? 'skip');
let allPass = true;
let allFailDueToUnresolved = true;
for (const c of criteria) {
if (c.type === 'loop_completion') {
// Can't simulate without loop tracking; treat as resolvable at runtime
continue;
}
const variable = c.variable;
if (typeof variable !== 'string') {
allFailDueToUnresolved = false;
continue;
}
const exprs = [...variable.matchAll(TEMPLATE_RE)];
if (exprs.length === 0) {
allFailDueToUnresolved = false;
continue;
}
const expr = exprs[0][1];
const res = resolveExpression(expr, state);
if (!res.resolved) {
// Unresolvable means we can't evaluate; assume runtime determines
continue;
}
allFailDueToUnresolved = false;
const operator = String(c.operator ?? '==');
const passes = evaluateOperator(res.value, operator, c.value);
if (!passes) {
allPass = false;
}
}
if (allPass)
return { run: true };
if (allFailDueToUnresolved)
return { run: true };
return {
run: false,
kind: onFail === 'stop' ? 'always-stop' : onFail === 'wait' ? 'always-wait' : 'always-skip',
reason: `entryConditions.criteria evaluated to false; onCriteriaFail=${onFail}`,
};
}
function evaluateOperator(left, op, right) {
switch (op) {
case '==':
case 'eq': return left === right;
case '!=':
case 'ne': return left !== right;
case '>': return Number(left) > Number(right);
case '<': return Number(left) < Number(right);
case '>=': return Number(left) >= Number(right);
case '<=': return Number(left) <= Number(right);
case 'isNull': return left === null || left === undefined;
case 'isNotNull': return left !== null && left !== undefined;
case 'isEmpty':
return left === '' || (Array.isArray(left) && left.length === 0)
|| (typeof left === 'object' && left !== null && Object.keys(left).length === 0);
case 'notEmpty':
return !(left === '' || (Array.isArray(left) && left.length === 0)
|| (typeof left === 'object' && left !== null && Object.keys(left).length === 0));
case 'contains':
return typeof left === 'string' && typeof right === 'string' && left.includes(right);
default:
return true; // unknown operator → assume runtime decides
}
}
// ---------------------------------------------------------------------------
// Credit estimate
// ---------------------------------------------------------------------------
const DEFAULT_CREDITS_BY_TYPE = {
aiAction: 10,
aiActionWithTools: 15,
appAction: 1,
code: 0,
setVariables: 0,
knowledgeSync: 1,
trigger: 0,
return: 0,
milestone: 0,
};
function creditsForStep(step) {
const explicit = Number(step.creditCost);
const baseCredit = Number.isFinite(explicit) && explicit >= 0
? explicit
: (DEFAULT_CREDITS_BY_TYPE[String(step.type ?? '')] ?? 0);
if (baseCredit <= 0 || !isBatchedStep(step))
return baseCredit;
const sourceLimit = extractBatchSourceLimit(step.batchConfig?.sourceVariable);
if (sourceLimit == null)
return baseCredit;
const batchSize = Number(step.batchConfig?.batchSize ?? 20);
const normalizedBatchSize = Number.isFinite(batchSize) && batchSize > 0 ? batchSize : 20;
return baseCredit * Math.ceil(sourceLimit / normalizedBatchSize);
}
function isBatchedStep(step) {
const batchConfig = step.batchConfig;
return batchConfig?.enabled === true;
}
function extractBatchSourceLimit(sourceVariable) {
const raw = Array.isArray(sourceVariable) ? sourceVariable.join('\n') : String(sourceVariable ?? '');
const match = raw.match(/\blimit\s*:\s*(\d+)\b/);
if (!match)
return undefined;
const limit = Number(match[1]);
return Number.isFinite(limit) && limit > 0 ? limit : undefined;
}
// ---------------------------------------------------------------------------
// Mock loaders
// ---------------------------------------------------------------------------
function loadMocksFromExecution(executionId) {
const wsDir = findWorkspaceDir();
const dir = wsDir
? join(wsDir, 'fixtures', 'step-outputs', executionId)
: resolve('fixtures', 'step-outputs', executionId);
if (!existsSync(dir)) {
throw new Error(`Fixture directory not found: ${dir}. Run "agentled fixture capture ${executionId} --wf <wfId>" first.`);
}
const result = {};
for (const file of readdirSync(dir).filter((f) => f.endsWith('.json'))) {
const stepId = file.replace(/\.json$/, '');
try {
const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
result[stepId] = data.output ?? data.stepOutput ?? data.result ?? data;
}
catch { /* skip malformed file */ }
}
return result;
}
function loadAppsCache() {
const wsDir = findWorkspaceDir();
if (!wsDir)
return null;
const cachePath = join(wsDir, '.agentled', 'cache', 'apps.json');
if (!existsSync(cachePath))
return null;
try {
return JSON.parse(readFileSync(cachePath, 'utf-8'));
}
catch {
return null;
}
}
// ---------------------------------------------------------------------------
// Walker
// ---------------------------------------------------------------------------
function walkPipeline(pipeline, overrides) {
const allSteps = steps(pipeline);
const trigger = findTrigger(allSteps);
if (!trigger)
return { steps: [], terminal: { reached: false } };
const state = {
input: overrides.input,
steps: { ...overrides.mocks },
context: synthesizeContext(pipeline),
};
const visited = new Set();
const trace = [];
let cursorId = String(trigger.id ?? '');
let terminalType;
let terminalStepId;
while (cursorId) {
if (visited.has(cursorId))
break; // cycle safety
visited.add(cursorId);
const step = findStepById(allSteps, cursorId);
if (!step)
break;
const type = String(step.type ?? '');
const ec = simulateEntryConditions(step, state);
const credit = creditsForStep(step);
let mock;
const unresolved = [];
if (cursorId in state.steps) {
// Caller-provided mock takes precedence
mock = state.steps[cursorId];
}
else {
mock = synthesizeMockOutput(step, overrides.appsCache);
}
// Check templates in stepInputData / pipelineStepPrompt.template / setVariables / knowledgeSync
const fieldsToCheck = [];
if (step.stepInputData)
fieldsToCheck.push({ value: step.stepInputData, field: 'stepInputData' });
const prompt = step.pipelineStepPrompt;
if (prompt?.template)
fieldsToCheck.push({ value: prompt.template, field: 'pipelineStepPrompt.template' });
if (step.setVariablesConfig)
fieldsToCheck.push({ value: step.setVariablesConfig, field: 'setVariablesConfig' });
if (step.knowledgeSync)
fieldsToCheck.push({ value: step.knowledgeSync, field: 'knowledgeSync' });
if (step.returnConfig)
fieldsToCheck.push({ value: step.returnConfig, field: 'returnConfig' });
for (const f of fieldsToCheck) {
findExpressionsAndCheck(f.value, state, cursorId, f.field, unresolved);
}
let status = 'executed';
let skipReason;
if (!ec.run) {
status = ec.kind === 'always-stop' ? 'stop' : ec.kind === 'always-wait' ? 'wait' : 'skipped';
skipReason = ec.reason;
}
// Record the synthesized output into state for downstream resolution
if (status === 'executed' || status === 'wait') {
state.steps[cursorId] = mock;
}
trace.push({
stepId: cursorId,
type,
name: step.name,
status,
creditCost: status === 'executed' ? credit : 0,
mockOutput: mock,
unresolved,
skipReason,
});
if (type === 'milestone' || type === 'return') {
terminalType = type;
terminalStepId = cursorId;
break;
}
if (status === 'stop' || status === 'wait')
break;
cursorId = getNextStepId(step);
}
return {
steps: trace,
terminal: { reached: !!terminalType, type: terminalType, stepId: terminalStepId },
};
}
// ---------------------------------------------------------------------------
// Command
// ---------------------------------------------------------------------------
export function registerDryRunCommand(program) {
program
.command('dryrun <file>')
.alias('dry-run')
.description('Zero-credit variable-resolution walk over a pipeline JSON. Synthesizes mock outputs per step (or uses --mocks / --from-execution), resolves all {{...}} references, and reports unresolved refs, always-skip branches, and a credit estimate. Run before `workflows start` to catch data-flow bugs without spending credits.')
.option('--input <json>', 'Inline JSON object for {{input.*}} resolution')
.option('--input-file <path>', 'Path to a JSON file containing the input object')
.option('--mocks <path>', 'Path to a JSON file mapping {stepId: <output>} to override synthetic mocks')
.option('--from-execution <execId>', 'Load step outputs from fixtures/step-outputs/<execId>/ (most realistic — pair with `agentled fixture capture`)')
.option('--format <fmt>', 'Output format', 'json')
.action((file, opts) => {
try {
const abs = resolve(file);
if (!existsSync(abs)) {
printError(`File not found: ${abs}`);
process.exitCode = VALIDATION_EXIT_CODE;
return;
}
let pipeline;
try {
pipeline = JSON.parse(readFileSync(abs, 'utf-8'));
}
catch (err) {
printError(`Failed to parse pipeline JSON: ${err instanceof Error ? err.message : String(err)}`);
process.exitCode = VALIDATION_EXIT_CODE;
return;
}
// Resolve input
let input = synthesizeInput(pipeline);
let inputProvided = false;
if (opts.input) {
try {
input = { ...input, ...JSON.parse(opts.input) };
inputProvided = true;
}
catch (err) {
printError(`Failed to parse --input JSON: ${err instanceof Error ? err.message : String(err)}`);
process.exitCode = VALIDATION_EXIT_CODE;
return;
}
}
if (opts.inputFile) {
try {
const raw = readFileSync(resolve(opts.inputFile), 'utf-8');
input = { ...input, ...JSON.parse(raw) };
inputProvided = true;
}
catch (err) {
printError(`Failed to read --input-file: ${err instanceof Error ? err.message : String(err)}`);
process.exitCode = VALIDATION_EXIT_CODE;
return;
}
}
// Resolve mocks
let mocks = {};
if (opts.mocks) {
try {
mocks = JSON.parse(readFileSync(resolve(opts.mocks), 'utf-8'));
}
catch (err) {
printError(`Failed to read --mocks file: ${err instanceof Error ? err.message : String(err)}`);
process.exitCode = VALIDATION_EXIT_CODE;
return;
}
}
if (opts.fromExecution) {
try {
const execMocks = loadMocksFromExecution(opts.fromExecution);
mocks = { ...mocks, ...execMocks };
}
catch (err) {
printError(err instanceof Error ? err.message : String(err));
process.exitCode = VALIDATION_EXIT_CODE;
return;
}
}
const appsCache = loadAppsCache();
const walked = walkPipeline(pipeline, { input, mocks, appsCache });
const unresolvedRefs = walked.steps.flatMap((s) => s.unresolved);
const branchIssues = walked.steps
.filter((s) => s.status !== 'executed')
.map((s) => ({
stepId: s.stepId,
kind: s.status === 'stop' ? 'always-stop'
: s.status === 'wait' ? 'always-wait'
: 'always-skip',
onCriteriaFail: s.skipReason,
}));
const creditEstimate = walked.steps.reduce((acc, s) => acc + s.creditCost, 0);
const result = {
file: abs,
pipeline: { name: pipeline.name, stepCount: steps(pipeline).length },
inputProvided,
mocksProvided: Object.keys(mocks).length,
fromExecution: opts.fromExecution,
steps: walked.steps,
unresolvedRefs,
branchIssues,
creditEstimate,
terminal: walked.terminal,
};
printOutput(result, (opts.format ?? 'json'));
const unresolvedErrors = unresolvedRefs.filter((r) => r.severity === 'error');
const unresolvedWarnings = unresolvedRefs.filter((r) => r.severity === 'warning');
if (unresolvedErrors.length > 0)
process.exitCode = VALIDATION_EXIT_CODE;
else if (unresolvedWarnings.length > 0 || branchIssues.length > 0 || !walked.terminal.reached)
process.exitCode = WARNING_EXIT_CODE;
}
catch (e) {
printError(e instanceof Error ? e.message : String(e));
}
});
}
//# sourceMappingURL=dryrun.js.map