arela/test-openai-vs-ollama.mjs

AI-powered CTO with multi-agent orchestration, code summarization, visual testing (web + mobile) for blazing fast development.

#!/usr/bin/env node
/**
 * Compare OpenAI vs Ollama classification performance
 * Run: node test-openai-vs-ollama.mjs
 */

import { config } from 'dotenv';
import { QueryClassifier } from './dist/meta-rag/classifier.js';

// Load .env file
config();

const queries = [
  "Continue working on authentication",
  "What is JWT?",
  "Show me auth dependencies",
  "What's my preferred testing framework?",
  "Why did we choose Postgres?"
];

console.log('🧪 Comparing OpenAI vs Ollama Classification\n');

// Test with OpenAI
console.log('📊 Testing OpenAI (gpt-4o-mini)...\n');
const openaiClassifier = new QueryClassifier();
await openaiClassifier.init();

const openaiTimes = [];
for (const query of queries) {
  const start = Date.now();
  const result = await openaiClassifier.classify(query);
  const duration = Date.now() - start;
  openaiTimes.push(duration);
  console.log(`  "${query}"`);
  console.log(`  → ${result.type} (${result.confidence}) in ${duration}ms\n`);
}

const openaiAvg = Math.round(openaiTimes.reduce((a, b) => a + b, 0) / openaiTimes.length);
const openaiMin = Math.min(...openaiTimes);
const openaiMax = Math.max(...openaiTimes);

console.log(`\n📈 OpenAI Stats:`);
console.log(`  Average: ${openaiAvg}ms`);
console.log(`  Min: ${openaiMin}ms`);
console.log(`  Max: ${openaiMax}ms`);
console.log(`  Cost: ~$0.0005 (5 queries)\n`);

console.log('✅ OpenAI is working great!\n');
console.log('💡 Tip: 700-1500ms is normal for API calls');
console.log('💡 This is faster than querying all 6 memory layers!');