claude-flow
Version:
Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration
423 lines (396 loc) • 19.5 kB
JavaScript
// Verified-corpus benchmark for ruflo-cost-tracker.
// Runs every case in bench/booster-corpus.json through agent-booster.apply()
// and records: correctness vs. golden expected, latency, confidence, strategy.
// Output: JSON to docs/benchmarks/runs/<ISO>.json + markdown summary to stdout.
//
// Resolution: must be run from a directory where `agent-booster` resolves
// (typically anywhere under `v3/`). Run via:
//
// ( cd v3 && node ../plugins/ruflo-cost-tracker/scripts/bench.mjs )
//
// Optional env:
// BENCH_LLM_BASELINE=1 -- run the same corpus through Gemini 2.0 Flash (or another OpenAI-compat model)
// BENCH_LLM_MODEL=... -- override (default: models/gemini-2.0-flash)
// BENCH_LLM_BASE_URL=... -- override (default: deployed ruvocal Gemini OpenAI shim)
// BENCH_LLM_API_KEY=... -- override (default: from gcloud secret GOOGLE_AI_API_KEY)
// BENCH_LLM_PRICE_IN/OUT -- $/1M tokens override (default Gemini 2.0 Flash: 0.10 / 0.40)
//
// BENCH_ANTHROPIC=1 -- run the same corpus through Anthropic claude models
// BENCH_ANTHROPIC_MODELS=claude-sonnet-4-6,claude-opus-4-7 -- comma-separated list
// BENCH_ANTHROPIC_API_KEY=... -- override (default: from gcloud secret ANTHROPIC_API_KEY)
//
// BENCH_OUT=<path> -- override output JSON path
// BENCH_QUIET=1 -- suppress markdown summary
//
// Pricing built-in (per 1M tokens, USD): Sonnet 4.6 = 3/15, Opus 4.7 = 15/75, Haiku 4.5 = 1/5.
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { createRequire } from 'node:module';
import { dirname, join, resolve } from 'node:path';
const HERE = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = resolve(HERE, '..');
const CORPUS = join(PLUGIN_ROOT, 'bench', 'booster-corpus.json');
const RUNS_DIR = join(PLUGIN_ROOT, 'docs', 'benchmarks', 'runs');
const norm = (s) => String(s ?? '').trim().replace(/\s+/g, ' ');
const pct = (n) => `${(n * 100).toFixed(1)}%`;
const ms = (n) => `${n.toFixed(2)}ms`;
const pcent = (arr, p) => {
if (!arr.length) return 0;
const sorted = [...arr].sort((a, b) => a - b);
const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
return sorted[idx];
};
async function main() {
const corpus = JSON.parse(readFileSync(CORPUS, 'utf-8'));
// Resolve agent-booster from process.cwd() (so users can `cd v3 && node ../...`)
// rather than from the script's directory (which is outside v3/node_modules).
let AgentBoosterMod;
try {
const requireFromCwd = createRequire(join(process.cwd(), 'package.json'));
const resolvedPath = requireFromCwd.resolve('agent-booster');
AgentBoosterMod = await import(pathToFileURL(resolvedPath).href);
} catch (err) {
console.error(`agent-booster import failed: ${err.message}`);
console.error(`cwd was: ${process.cwd()}`);
console.error('Run from a directory where the package resolves, e.g.:');
console.error(' ( cd v3 && node ../plugins/ruflo-cost-tracker/scripts/bench.mjs )');
process.exit(2);
}
const { AgentBooster } = AgentBoosterMod;
const booster = new AgentBooster();
const results = [];
for (const c of corpus.cases) {
const expectedTier1 = c.expectedTier1 !== false; // default true (corpus v1 compat)
const t0 = Date.now();
let out;
try {
out = await booster.apply({ code: c.code, edit: c.edit, language: c.language });
} catch (e) {
results.push({ id: c.id, intent: c.intent, expectedTier1, error: String(e.message).slice(0, 200), correct: false });
continue;
}
const wallMs = Date.now() - t0;
const correct = norm(out.output) === norm(c.expected);
const lowConfidence = (out.confidence ?? 0) < 0.5;
// For Tier 1 cases: "good" = correct.
// For non-Tier 1 cases: "good" = booster correctly *escalates* (incorrect output OR low confidence).
const escalatedCorrectly = !expectedTier1 && (!correct || lowConfidence);
results.push({
id: c.id,
intent: c.intent,
expectedTier1,
success: !!out.success,
correct,
escalatedCorrectly,
lowConfidence,
latencyMs: out.latency ?? null,
wallMs,
confidence: out.confidence ?? null,
strategy: out.strategy ?? null,
tokensIn: out.tokens?.input ?? null,
tokensOut: out.tokens?.output ?? null,
...(correct ? {} : { actualPrefix: norm(out.output).slice(0, 120), expectedPrefix: norm(c.expected).slice(0, 120) }),
});
}
const tier1Cases = results.filter((r) => r.expectedTier1);
const advCases = results.filter((r) => !r.expectedTier1);
const tier1Passed = tier1Cases.filter((r) => r.correct).length;
const tier1Total = tier1Cases.length;
const advEscalated = advCases.filter((r) => r.escalatedCorrectly).length;
const advTotal = advCases.length;
const passed = results.filter((r) => r.correct).length;
const total = results.length;
const winRate = tier1Total ? tier1Passed / tier1Total : (total ? passed / total : 0);
const escalationRate = advTotal ? advEscalated / advTotal : null;
const successCount = results.filter((r) => r.success).length;
const latencies = results.map((r) => r.latencyMs).filter((x) => typeof x === 'number');
const wallTimes = results.map((r) => r.wallMs).filter((x) => typeof x === 'number');
const confidences = results.map((r) => r.confidence).filter((x) => typeof x === 'number');
const avg = (arr) => (arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : 0);
// Optional: run the same corpus through one or more LLMs and record the comparison.
let llmSummary = null;
let llmResults = null;
if (process.env.BENCH_LLM_BASELINE === '1') {
({ llmSummary, llmResults } = await runLlmBaseline(corpus.cases));
}
let anthropicSummaries = null; // Map<model, summary>
let anthropicResults = null; // Map<model, per-case-results>
if (process.env.BENCH_ANTHROPIC === '1') {
({ anthropicSummaries, anthropicResults } = await runAnthropicBaseline(corpus.cases));
}
const summary = {
runAt: new Date().toISOString(),
corpusVersion: corpus.version,
corpusSize: total,
tier1Cases: tier1Total,
adversarialCases: advTotal,
winRate, // win rate over Tier 1 cases (the gate metric)
winRatePct: pct(winRate),
overallCorrect: total ? passed / total : 0, // diagnostic only
escalationRate, // null if no adversarial cases; otherwise (correctly-escalated / adversarial)
escalationRatePct: escalationRate == null ? 'n/a' : pct(escalationRate),
successCount,
avgLatencyMs: avg(latencies),
p50LatencyMs: pcent(latencies, 50),
p99LatencyMs: pcent(latencies, 99),
maxLatencyMs: latencies.length ? Math.max(...latencies) : 0,
avgWallMs: avg(wallTimes),
avgConfidence: avg(confidences),
minConfidence: confidences.length ? Math.min(...confidences) : 0,
confidenceThreshold: 0.5,
aboveThresholdCount: confidences.filter((c) => c >= 0.5).length,
structuralCostUsd: 0, // no LLM call → no billing
llmBaseline: llmSummary || (process.env.BENCH_LLM_BASELINE === '1' ? 'enabled-but-failed' : 'skipped'),
};
if (llmSummary) {
// Direct apples-to-apples speedup ratio (booster vs LLM)
summary.speedupVsLlm = llmSummary.avgLatencyMs / Math.max(summary.avgLatencyMs, 0.001);
summary.costDeltaUsdPerEdit = llmSummary.avgCostUsdPerEdit; // booster side is $0
}
if (anthropicSummaries) {
summary.anthropic = {};
for (const [model, s] of Object.entries(anthropicSummaries)) {
summary.anthropic[model] = {
...s,
speedupVsBooster: s.avgLatencyMs / Math.max(summary.avgLatencyMs, 0.001),
costSavedPerEditUsd: s.avgCostUsdPerEdit, // booster side is $0
};
}
}
const outDir = RUNS_DIR;
mkdirSync(outDir, { recursive: true });
const stamp = summary.runAt.replace(/[:.]/g, '-');
const outPath = process.env.BENCH_OUT || join(outDir, `${stamp}.json`);
const latestPath = join(outDir, 'latest.json');
const payload = {
summary,
results,
...(llmResults ? { llmResults } : {}),
...(anthropicResults ? { anthropicResults } : {}),
};
writeFileSync(outPath, JSON.stringify(payload, null, 2));
writeFileSync(latestPath, JSON.stringify(payload, null, 2));
if (process.env.BENCH_QUIET !== '1') {
console.log(`# Booster benchmark — ${summary.runAt}`);
console.log('');
console.log(`| Metric | Value |`);
console.log(`|---|---:|`);
console.log(`| Corpus size | ${total} (Tier 1: ${tier1Total}, adversarial: ${advTotal}) |`);
console.log(`| **Win rate (Tier 1 cases)** | **${pct(winRate)}** (${tier1Passed}/${tier1Total}) |`);
if (advTotal) {
console.log(`| Escalation rate (adversarial) | ${pct(escalationRate)} (${advEscalated}/${advTotal}) |`);
}
console.log(`| Overall correct (diagnostic) | ${pct(total ? passed/total : 0)} (${passed}/${total}) |`);
console.log(`| Success flag | ${successCount}/${total} |`);
console.log(`| Avg latency | ${ms(summary.avgLatencyMs)} |`);
console.log(`| p50 latency | ${ms(summary.p50LatencyMs)} |`);
console.log(`| p99 latency | ${ms(summary.p99LatencyMs)} |`);
console.log(`| Max latency | ${ms(summary.maxLatencyMs)} |`);
console.log(`| Avg confidence | ${summary.avgConfidence.toFixed(3)} |`);
console.log(`| Min confidence | ${summary.minConfidence.toFixed(3)} |`);
console.log(`| Above 0.5 threshold | ${summary.aboveThresholdCount}/${confidences.length} |`);
console.log(`| Structural cost | $${summary.structuralCostUsd} (no LLM call) |`);
if (llmSummary) {
console.log(`| LLM baseline model | ${llmSummary.model} |`);
console.log(`| LLM avg latency | ${ms(llmSummary.avgLatencyMs)} |`);
console.log(`| LLM win rate | ${pct(llmSummary.winRate)} (${llmSummary.passed}/${llmSummary.total}) |`);
console.log(`| LLM avg cost/edit | $${llmSummary.avgCostUsdPerEdit.toFixed(6)} |`);
console.log(`| **Measured speedup (booster vs LLM)** | **${summary.speedupVsLlm.toFixed(1)}×** |`);
console.log(`| **Cost saved per edit** | **$${summary.costDeltaUsdPerEdit.toFixed(6)}** (100%) |`);
} else {
console.log(`| LLM baseline | ${summary.llmBaseline} |`);
}
console.log(``);
if (anthropicSummaries) {
console.log(`## Anthropic baseline\n`);
console.log(`| Model | Avg latency | Win rate | Avg tokens (in/out) | Avg cost/edit | Speedup vs booster | Cost saved/edit |`);
console.log(`|---|---:|---:|---:|---:|---:|---:|`);
for (const [model, s] of Object.entries(anthropicSummaries)) {
const speedup = (s.avgLatencyMs / Math.max(summary.avgLatencyMs, 0.001)).toFixed(1);
console.log(`| \`${model}\` | ${ms(s.avgLatencyMs)} | ${pct(s.winRate)} (${s.passed}/${s.total}) | ${s.avgTokensIn.toFixed(0)} / ${s.avgTokensOut.toFixed(0)} | $${s.avgCostUsdPerEdit.toFixed(6)} | **${speedup}×** | **$${s.avgCostUsdPerEdit.toFixed(6)}** |`);
}
console.log(``);
}
const failed = results.filter((r) => !r.correct);
if (failed.length) {
console.log(`## Failures (${failed.length})`);
for (const f of failed) {
console.log(`- \`${f.id}\` (${f.intent}): ${f.error || `actual="${f.actualPrefix}" vs expected="${f.expectedPrefix}"`}`);
}
}
console.log(``);
console.log(`Saved: ${outPath}`);
console.log(`Latest pointer: ${latestPath}`);
}
}
// ─── LLM baseline ────────────────────────────────────────────────────────────
async function runLlmBaseline(cases) {
const baseUrl = process.env.BENCH_LLM_BASE_URL || 'https://generativelanguage.googleapis.com/v1beta/openai/';
const model = process.env.BENCH_LLM_MODEL || 'models/gemini-2.0-flash';
const priceIn = parseFloat(process.env.BENCH_LLM_PRICE_IN || '0.10'); // $/1M input
const priceOut = parseFloat(process.env.BENCH_LLM_PRICE_OUT || '0.40'); // $/1M output
let apiKey = process.env.BENCH_LLM_API_KEY;
if (!apiKey) {
// Try to pull from gcloud (deployed ruvocal uses GOOGLE_AI_API_KEY)
try {
const { execSync } = await import('node:child_process');
apiKey = execSync('gcloud secrets versions access latest --secret=GOOGLE_AI_API_KEY 2>/dev/null', { encoding: 'utf-8' }).trim();
} catch { /* fall through */ }
}
if (!apiKey) {
return { llmSummary: { error: 'no-api-key', baseUrl, model }, llmResults: [] };
}
const sys = `You apply code edits deterministically. Return ONLY the resulting code as a single fenced \`\`\`<lang> code block. No explanation, no commentary, no extra blocks. The output of the code block is the final source.`;
const user = (c) => `Apply this edit. Return only the resulting code.\n\nLanguage: ${c.language}\n\nOriginal code:\n\`\`\`${c.language}\n${c.code}\n\`\`\`\n\nEdit instruction (target snippet):\n\`\`\`${c.language}\n${c.edit}\n\`\`\``;
const fenceRe = /```(?:[a-zA-Z]+\n)?([\s\S]*?)```/;
const out = [];
let totIn = 0, totOut = 0, totLatencyMs = 0, passed = 0;
for (const c of cases) {
const t0 = Date.now();
let body = null;
try {
const resp = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
method: 'POST',
headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
messages: [{ role: 'system', content: sys }, { role: 'user', content: user(c) }],
max_tokens: 1024,
temperature: 0,
}),
});
body = await resp.json();
} catch (e) {
out.push({ id: c.id, intent: c.intent, error: String(e.message).slice(0, 200), correct: false, wallMs: Date.now() - t0 });
continue;
}
const wallMs = Date.now() - t0;
const content = body?.choices?.[0]?.message?.content ?? '';
const m = fenceRe.exec(content);
const extracted = (m ? m[1] : content).trim();
const correct = norm(extracted) === norm(c.expected);
if (correct) passed++;
const inT = body?.usage?.prompt_tokens ?? 0;
const outT = body?.usage?.completion_tokens ?? 0;
totIn += inT; totOut += outT; totLatencyMs += wallMs;
const cost = inT / 1e6 * priceIn + outT / 1e6 * priceOut;
out.push({
id: c.id, intent: c.intent, correct, wallMs,
tokensIn: inT, tokensOut: outT, costUsd: cost,
...(correct ? {} : { actualPrefix: norm(extracted).slice(0, 120), expectedPrefix: norm(c.expected).slice(0, 120) }),
});
}
const total = cases.length;
const totalCost = out.reduce((s, r) => s + (r.costUsd || 0), 0);
return {
llmSummary: {
model, baseUrl, total, passed,
winRate: total ? passed / total : 0,
avgLatencyMs: total ? totLatencyMs / total : 0,
totalTokensIn: totIn,
totalTokensOut: totOut,
avgTokensIn: total ? totIn / total : 0,
avgTokensOut: total ? totOut / total : 0,
totalCostUsd: totalCost,
avgCostUsdPerEdit: total ? totalCost / total : 0,
pricing: { input_per_1M: priceIn, output_per_1M: priceOut },
},
llmResults: out,
};
}
// ─── Anthropic baseline ──────────────────────────────────────────────────────
// Built-in pricing per 1M tokens (USD). Override with BENCH_ANTHROPIC_PRICING JSON env.
const ANTHROPIC_PRICING = {
'claude-sonnet-4-6': { input: 3.00, output: 15.00 },
'claude-opus-4-7': { input: 15.00, output: 75.00 },
'claude-haiku-4-5': { input: 1.00, output: 5.00 },
'claude-haiku-4-5-20251001': { input: 1.00, output: 5.00 },
};
async function runAnthropicBaseline(cases) {
let apiKey = process.env.BENCH_ANTHROPIC_API_KEY;
if (!apiKey) {
try {
const { execSync } = await import('node:child_process');
apiKey = execSync('gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY 2>/dev/null', { encoding: 'utf-8' }).trim();
} catch { /* fall through */ }
}
if (!apiKey) {
return { anthropicSummaries: { _error: 'no-api-key' }, anthropicResults: {} };
}
let pricingOverride = {};
if (process.env.BENCH_ANTHROPIC_PRICING) {
try { pricingOverride = JSON.parse(process.env.BENCH_ANTHROPIC_PRICING); } catch { /* ignore */ }
}
const models = (process.env.BENCH_ANTHROPIC_MODELS || 'claude-sonnet-4-6,claude-opus-4-7')
.split(',').map((s) => s.trim()).filter(Boolean);
const sys = `You apply code edits deterministically. Return ONLY the resulting code as a single fenced \`\`\`<lang> code block. No explanation, no commentary, no extra blocks.`;
const user = (c) => `Apply this edit. Return only the resulting code.\n\nLanguage: ${c.language}\n\nOriginal code:\n\`\`\`${c.language}\n${c.code}\n\`\`\`\n\nEdit instruction (target snippet):\n\`\`\`${c.language}\n${c.edit}\n\`\`\``;
const fenceRe = /```(?:[a-zA-Z]+\n)?([\s\S]*?)```/;
const summaries = {};
const allResults = {};
for (const model of models) {
const pricing = pricingOverride[model] || ANTHROPIC_PRICING[model] || { input: 3, output: 15 };
const out = [];
let totIn = 0, totOut = 0, totLatencyMs = 0, passed = 0;
for (const c of cases) {
const t0 = Date.now();
let body = null;
try {
const resp = await fetch('https://api.anthropic.com/v1/messages', {
method: 'POST',
headers: {
'x-api-key': apiKey,
'anthropic-version': '2023-06-01',
'Content-Type': 'application/json',
},
body: JSON.stringify({
model,
max_tokens: 1024,
system: sys,
messages: [{ role: 'user', content: user(c) }],
}),
});
body = await resp.json();
} catch (e) {
out.push({ id: c.id, intent: c.intent, error: String(e.message).slice(0, 200), correct: false, wallMs: Date.now() - t0 });
continue;
}
const wallMs = Date.now() - t0;
const text = body?.content?.[0]?.text ?? '';
const m = fenceRe.exec(text);
const extracted = (m ? m[1] : text).trim();
const correct = norm(extracted) === norm(c.expected);
if (correct) passed++;
const inT = body?.usage?.input_tokens ?? 0;
const outT = body?.usage?.output_tokens ?? 0;
totIn += inT; totOut += outT; totLatencyMs += wallMs;
const cost = inT / 1e6 * pricing.input + outT / 1e6 * pricing.output;
out.push({
id: c.id, intent: c.intent, correct, wallMs,
tokensIn: inT, tokensOut: outT, costUsd: cost,
...(correct ? {} : { actualPrefix: norm(extracted).slice(0, 120), expectedPrefix: norm(c.expected).slice(0, 120) }),
});
}
const total = cases.length;
const totalCost = out.reduce((s, r) => s + (r.costUsd || 0), 0);
summaries[model] = {
model, total, passed,
winRate: total ? passed / total : 0,
avgLatencyMs: total ? totLatencyMs / total : 0,
totalTokensIn: totIn,
totalTokensOut: totOut,
avgTokensIn: total ? totIn / total : 0,
avgTokensOut: total ? totOut / total : 0,
totalCostUsd: totalCost,
avgCostUsdPerEdit: total ? totalCost / total : 0,
pricing,
};
allResults[model] = out;
}
return { anthropicSummaries: summaries, anthropicResults: allResults };
}
main().catch((e) => {
console.error('bench failed:', e);
process.exit(1);
});