claude-flow
Version:
Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration
169 lines • 7.27 kB
JavaScript
/**
* Smoke tests for image_describe — ADR-133-PR5
*
* Run with:
* cd v3/@claude-flow/cli
* npx tsx src/benchmarks/gaia-tools/image_describe.smoke.ts
*
* Tests:
* 1. URL image (Wikipedia logo PNG) — description contains 'wiki' or 'logo' or 'W'
* 2. Missing API key — returns error string, does not crash
* 3. Non-existent local file — returns error string, does not crash
* 4. Invalid source (empty) — throws (caught by test harness)
*
* If ANTHROPIC_API_KEY is not set and gcloud is not available, Test 1 is
* marked SKIP (not FAIL) — the tool's graceful error path is validated in
* Test 2 instead.
*
* Cost estimate: ~$0.001 per live API call (Haiku vision, single small image).
* Total smoke cost: ≤$0.001 (only Test 1 makes a live call).
*
* Refs: ADR-133, #2156
*/
import { createImageDescribeTool } from './image_describe.js';
async function runTest(name, fn) {
try {
await fn();
return { name, status: 'PASS', message: 'OK' };
}
catch (e) {
return { name, status: 'FAIL', message: String(e) };
}
}
function assert(condition, msg) {
if (!condition)
throw new Error(`Assertion failed: ${msg}`);
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
// Test 1: Live API call — describe Wikipedia logo via URL.
// SKIP if no API key is available (avoids failing in CI without credentials).
async function testWikipediaLogo() {
const hasApiKey = (process.env.ANTHROPIC_API_KEY ?? '').trim().length > 0;
if (!hasApiKey) {
// Try gcloud as a secondary check — if both are unavailable, skip.
let gcpAvailable = false;
try {
const { execSync } = await import('node:child_process');
const key = execSync('gcloud secrets versions access latest --secret=ANTHROPIC_API_KEY 2>/dev/null', { encoding: 'utf-8', timeout: 5_000 }).trim();
gcpAvailable = key.length > 0;
}
catch {
gcpAvailable = false;
}
if (!gcpAvailable) {
// Signal SKIP by throwing a message the runner recognises.
throw new Error('SKIP: ANTHROPIC_API_KEY not available — set env var to run live tests');
}
}
const tool = createImageDescribeTool();
const out = await tool.execute({
// Small, stable public PNG — Wikipedia's logo.
source: 'https://upload.wikimedia.org/wikipedia/commons/thumb/8/80/Wikipedia-logo-v2.svg/200px-Wikipedia-logo-v2.svg.png',
prompt: 'Describe the logo in one sentence.',
});
// Accept a graceful API error (e.g., rate limit) rather than failing the smoke.
if (out.startsWith('[image_describe error]')) {
// Surface as a warning but not a hard failure for CI without credentials.
console.log(` [warn] API call returned error: ${out}`);
return;
}
assert(out.includes('[image_describe:'), `Expected model tag in output, got:\n${out.slice(0, 300)}`);
const lowerOut = out.toLowerCase();
assert(lowerOut.includes('wiki') ||
lowerOut.includes('logo') ||
lowerOut.includes('globe') ||
lowerOut.includes('puzzle') ||
lowerOut.includes('encyclopedia'), `Expected description to mention logo/wiki/globe, got:\n${out.slice(0, 300)}`);
}
// Test 2: No API key supplied → returns error string without crashing.
async function testMissingApiKey() {
// Temporarily shadow the env var to simulate missing key.
const savedKey = process.env.ANTHROPIC_API_KEY;
delete process.env.ANTHROPIC_API_KEY;
try {
// Create tool with a dummy key that will be rejected by the API — but
// the resolve step won't throw because we pass an explicit (invalid) key.
// We want to confirm that a missing key returns an error string, not an
// exception. To test the "no key at all" path we must clear env AND pass
// no key.
// Use a blank API key — resolveAnthropicApiKey should throw internally,
// which execute() catches and returns as a string.
const tool = createImageDescribeTool({ apiKey: '' });
const out = await tool.execute({
source: 'https://example.com/image.png',
prompt: 'Describe.',
});
assert(typeof out === 'string', 'Expected string output when key is missing');
// Either an error message or a valid description — both are acceptable.
// The key assertion is that execute() did NOT throw.
}
finally {
if (savedKey !== undefined)
process.env.ANTHROPIC_API_KEY = savedKey;
}
}
// Test 3: Non-existent local file → returns error string without crashing.
async function testMissingLocalFile() {
const tool = createImageDescribeTool();
const out = await tool.execute({
source: '/tmp/__gaia_nonexistent_image_abc123.png',
});
assert(typeof out === 'string', 'Expected string output for missing file');
assert(out.startsWith('[image_describe error]'), `Expected error prefix for missing file, got:\n${out}`);
assert(out.includes('not found') || out.includes('Cannot read'), `Expected "not found" or "Cannot read" in error, got:\n${out}`);
}
// Test 4: Empty source → throws (caller contract violation).
async function testEmptySource() {
const tool = createImageDescribeTool();
let threw = false;
try {
await tool.execute({ source: '' });
}
catch {
threw = true;
}
assert(threw, 'Expected execute() to throw for empty source');
}
// ---------------------------------------------------------------------------
// Runner
// ---------------------------------------------------------------------------
async function main() {
console.log('image_describe smoke tests — ADR-133-PR5\n');
const tests = [
runTest('Test 1: URL image — Wikipedia logo (live API)', testWikipediaLogo),
runTest('Test 2: Missing API key — graceful error string', testMissingApiKey),
runTest('Test 3: Non-existent local file — graceful error', testMissingLocalFile),
runTest('Test 4: Empty source — throws', testEmptySource),
];
const results = await Promise.all(tests);
let passed = 0;
let failed = 0;
let skipped = 0;
for (const r of results) {
// Tests that throw 'SKIP:' are shown as skipped.
const isSkip = r.status === 'FAIL' && r.message.startsWith('SKIP:');
const displayStatus = isSkip ? 'SKIP' : r.status;
console.log(`[${displayStatus}] ${r.name}`);
if (displayStatus !== 'PASS') {
console.log(` ${r.message.slice(0, 300)}`);
}
if (isSkip)
skipped++;
else if (r.status === 'PASS')
passed++;
else
failed++;
}
console.log(`\nResults: ${passed} passed, ${failed} failed, ${skipped} skipped`);
console.log(`Cost estimate: ≤$0.001 per run (single Haiku vision call for Test 1)`);
if (failed > 0) {
process.exit(1);
}
}
main().catch((e) => {
console.error('Fatal:', e);
process.exit(1);
});
//# sourceMappingURL=image_describe.smoke.js.map