contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
161 lines (160 loc) • 8.22 kB
JavaScript
/**
* Phase 1 Behavior Test: Verify agent responds appropriately to different request types
*
* This test simulates different user requests and checks if the agent:
* 1. Doesn't assume story creation for generic requests
* 2. Uses appropriate tools for content analysis
* 3. Only enters story mode when explicitly requested
*/
import { ChatService } from '../services/chatService.js';
import { PromptTemplates } from '../services/prompts/PromptTemplates.js';
import fs from 'fs/promises';
import path from 'path';
async function testPhase1Behavior() {
console.log('🧪 Phase 1 Behavior Test: Agent Response Patterns\n');
console.log('This test verifies the agent uses the new generic system prompt.\n');
const testDir = path.join(process.cwd(), 'test_phase1_output');
try {
// Create test directory
await fs.mkdir(testDir, { recursive: true });
console.log(`✓ Created test directory: ${testDir}\n`);
// Initialize chat service
const chatService = new ChatService(testDir);
console.log('✓ Initialized ChatService\n');
// Test 1: Verify system prompt is generic
console.log('Test 1: System Prompt Verification');
console.log('='.repeat(60));
const systemPrompt = PromptTemplates.getDefaultSystemPrompt();
console.log('System prompt starts with:');
console.log(systemPrompt.substring(0, 200) + '...\n');
const isGeneric = systemPrompt.includes('generic content development platform');
const hasMultiModal = systemPrompt.includes('multi-modal capabilities');
const hasStateMachine = systemPrompt.includes('Story Project State Machine');
console.log(` ✓ Generic identity: ${isGeneric ? '✅' : '❌'}`);
console.log(` ✓ Multi-modal capabilities: ${hasMultiModal ? '✅' : '❌'}`);
console.log(` ✓ State machine docs: ${hasStateMachine ? '✅' : '❌'}`);
// Test 2: Create a session and verify it uses the new prompt
console.log('\nTest 2: Session Creation with New Prompt');
console.log('='.repeat(60));
const session = await chatService.createSession({
systemPrompt: PromptTemplates.getDefaultSystemPrompt()
});
console.log(` ✓ Session created: ${session.id}`);
console.log(` ✓ System prompt length: ${systemPrompt.length} chars`);
console.log(` ✓ Estimated tokens: ~${Math.ceil(systemPrompt.length / 4)}`);
// Test 3: Verify prompt includes key sections
console.log('\nTest 3: Key Sections Present');
console.log('='.repeat(60));
const keySections = [
{ name: 'Capabilities Section', text: '## Your Capabilities' },
{ name: 'File Operations', text: '### File Operations' },
{ name: 'Audio Tools', text: '### Audio Tools' },
{ name: 'Video & Image Tools', text: '### Video & Image Tools' },
{ name: 'Story Tools', text: '### Story & Screenplay Tools' },
{ name: 'Behavior Guidelines', text: '## Your Behavior' },
{ name: 'State Machine', text: '## Story Project State Machine' },
{ name: 'Intent Detection', text: '## Intent Detection' },
{ name: 'Examples', text: '## Examples - Multi-Modal Usage' },
{ name: 'Best Practices', text: '## Best Practices' }
];
keySections.forEach(section => {
const hasSection = systemPrompt.includes(section.text);
console.log(` ✓ ${section.name}: ${hasSection ? '✅' : '❌'}`);
});
// Test 4: Verify state machine states are documented
console.log('\nTest 4: State Machine States');
console.log('='.repeat(60));
const states = [
'OUTLINE_DRAFT',
'OUTLINE_CONFIRMED',
'SCREENPLAY_WRITING',
'ITERATION'
];
states.forEach(state => {
const hasState = systemPrompt.includes(state);
console.log(` ✓ ${state}: ${hasState ? '✅' : '❌'}`);
});
// Test 5: Verify intent detection categories
console.log('\nTest 5: Intent Detection Categories');
console.log('='.repeat(60));
const intentCategories = [
'Questions (No Workflow Trigger)',
'Content Analysis (Use Tools, No Story Workflow)',
'Story Project Work (Apply State Machine)',
'Explicit Requests (Always Honor)'
];
intentCategories.forEach(category => {
const hasCategory = systemPrompt.includes(category);
console.log(` ✓ ${category}: ${hasCategory ? '✅' : '❌'}`);
});
// Test 6: Verify tool examples
console.log('\nTest 6: Tool Examples');
console.log('='.repeat(60));
const toolExamples = [
{ tool: 'ffmpeg', example: '<tool_call name="ffmpeg"' },
{ tool: 'audio_generation', example: '<tool_call name="audio_generation"' },
{ tool: 'read_file', example: '<tool_call name="read_file"' },
{ tool: 'write_file', example: '<tool_call name="write_file"' },
{ tool: 'cinematic_workflow', example: '<tool_call name="cinematic_workflow"' }
];
toolExamples.forEach(({ tool, example }) => {
const hasExample = systemPrompt.includes(example);
console.log(` ✓ ${tool}: ${hasExample ? '✅' : '❌'}`);
});
// Test 7: Verify critical behavior guidelines
console.log('\nTest 7: Critical Behavior Guidelines');
console.log('='.repeat(60));
const criticalGuidelines = [
'DO NOT assume every request is about story creation',
'Answer questions directly without triggering workflows',
'Use appropriate tools based on user intent',
'Only enter story creation mode when explicitly requested'
];
criticalGuidelines.forEach(guideline => {
const hasGuideline = systemPrompt.includes(guideline);
console.log(` ✓ ${guideline.substring(0, 50)}...: ${hasGuideline ? '✅' : '❌'}`);
});
// Summary
console.log('\n' + '='.repeat(60));
console.log('📊 Phase 1 Behavior Test Summary');
console.log('='.repeat(60));
const allChecks = [
isGeneric,
hasMultiModal,
hasStateMachine,
...keySections.map(s => systemPrompt.includes(s.text)),
...states.map(s => systemPrompt.includes(s)),
...intentCategories.map(c => systemPrompt.includes(c)),
...toolExamples.map(t => systemPrompt.includes(t.example)),
...criticalGuidelines.map(g => systemPrompt.includes(g))
];
const passedChecks = allChecks.filter(c => c).length;
const totalChecks = allChecks.length;
const passRate = ((passedChecks / totalChecks) * 100).toFixed(1);
console.log(`\nChecks Passed: ${passedChecks}/${totalChecks} (${passRate}%)`);
if (passedChecks === totalChecks) {
console.log('\n✅ Phase 1 implementation is complete and correct!');
console.log('\nThe agent now:');
console.log(' • Has a generic content development platform identity');
console.log(' • Documents all multi-modal capabilities');
console.log(' • Includes story project state machine');
console.log(' • Provides intent detection guidelines');
console.log(' • Shows multi-modal usage examples');
console.log(' • Emphasizes NOT assuming story creation');
console.log('\n✨ Ready for Phase 2: State Detection & Management');
}
else {
console.log('\n⚠️ Some checks failed. Review the output above.');
}
// Cleanup
console.log(`\n🧹 Cleaning up test directory: ${testDir}`);
await fs.rm(testDir, { recursive: true, force: true });
console.log('✓ Cleanup complete\n');
}
catch (error) {
console.error('❌ Test failed with error:', error);
throw error;
}
}
// Run the test
testPhase1Behavior().catch(console.error);