knowledgegraph-mcp
Version:
MCP server for enabling persistent knowledge storage for Claude through a knowledge graph with multiple storage backends
259 lines ā¢ 10.9 kB
JavaScript
import { promises as fs } from 'fs';
import path from 'path';
import { PostgreSQLFuzzyStrategy } from '../../search/strategies/postgresql-strategy.js';
import { SQLiteFuzzyStrategy } from '../../search/strategies/sqlite-strategy.js';
import { PerformanceUtils } from './performance-utils.js';
/**
* Comprehensive benchmark runner for the knowledge graph MCP service
*/
export class BenchmarkRunner {
results = [];
/**
* Run all benchmarks and generate a comprehensive report
*/
async runAllBenchmarks() {
console.log('š Starting Knowledge Graph MCP Performance Benchmarks...\n');
// Client-side search benchmarks
await this.runClientSideSearchBenchmarks();
// Memory usage benchmarks
await this.runMemoryBenchmarks();
// Threshold comparison benchmarks
await this.runThresholdBenchmarks();
// Strategy comparison benchmarks
await this.runStrategyComparisonBenchmarks();
const report = this.generateReport();
await this.saveReport(report);
return report;
}
async runClientSideSearchBenchmarks() {
console.log('š Running Client-side Search Benchmarks...');
const config = {
useDatabaseSearch: false,
threshold: 0.3,
clientSideFallback: true
};
const datasets = [
{ name: 'Small (100 entities)', size: 100, iterations: 50 },
{ name: 'Medium (1000 entities)', size: 1000, iterations: 20 },
{ name: 'Large (10000 entities)', size: 10000, iterations: 10 }
];
for (const dataset of datasets) {
const entities = PerformanceUtils.generateRealisticTestEntities(dataset.size);
const mockPool = {};
const strategy = new PostgreSQLFuzzyStrategy(config, mockPool, 'test-project');
const benchmark = await PerformanceUtils.runBenchmark(async () => {
return strategy.searchClientSide(entities, 'JavaScript');
}, dataset.iterations);
this.results.push({
category: 'Client-side Search',
test: dataset.name,
avgTime: benchmark.avgTime,
minTime: benchmark.minTime,
maxTime: benchmark.maxTime,
medianTime: benchmark.medianTime,
stdDev: benchmark.stdDev,
iterations: dataset.iterations,
resultsCount: benchmark.results[0].length,
metadata: {
entityCount: dataset.size,
searchTerm: 'JavaScript'
}
});
console.log(` ā
${dataset.name}: ${benchmark.avgTime.toFixed(2)}ms avg`);
}
}
async runMemoryBenchmarks() {
console.log('\nš§ Running Memory Usage Benchmarks...');
const config = {
useDatabaseSearch: false,
threshold: 0.3,
clientSideFallback: true
};
const datasets = [
{ name: 'Small (100 entities)', size: 100 },
{ name: 'Large (10000 entities)', size: 10000 }
];
for (const dataset of datasets) {
const entities = PerformanceUtils.generateRealisticTestEntities(dataset.size);
const mockPool = {};
const strategy = new PostgreSQLFuzzyStrategy(config, mockPool, 'test-project');
const memoryTest = await PerformanceUtils.measureMemory(async () => {
return strategy.searchClientSide(entities, 'Node.js');
});
this.results.push({
category: 'Memory Usage',
test: dataset.name,
avgTime: 0, // Not applicable for memory tests
minTime: 0,
maxTime: 0,
medianTime: 0,
stdDev: 0,
iterations: 1,
resultsCount: memoryTest.result.length,
metadata: {
entityCount: dataset.size,
memoryUsed: memoryTest.memoryUsed,
heapBefore: memoryTest.heapUsedBefore,
heapAfter: memoryTest.heapUsedAfter,
searchTerm: 'Node.js'
}
});
console.log(` ā
${dataset.name}: ${PerformanceUtils.formatMemoryUsage(memoryTest.memoryUsed)} used`);
}
}
async runThresholdBenchmarks() {
console.log('\nšÆ Running Threshold Comparison Benchmarks...');
const entities = PerformanceUtils.generateRealisticTestEntities(1000);
const thresholds = [0.1, 0.3, 0.5, 0.7, 0.9];
for (const threshold of thresholds) {
const config = {
useDatabaseSearch: false,
threshold,
clientSideFallback: true
};
const mockPool = {};
const strategy = new PostgreSQLFuzzyStrategy(config, mockPool, 'test-project');
const benchmark = await PerformanceUtils.runBenchmark(async () => {
return strategy.searchClientSide(entities, 'JavaScrpt'); // Intentional typo
}, 20);
this.results.push({
category: 'Threshold Comparison',
test: `Threshold ${threshold}`,
avgTime: benchmark.avgTime,
minTime: benchmark.minTime,
maxTime: benchmark.maxTime,
medianTime: benchmark.medianTime,
stdDev: benchmark.stdDev,
iterations: 20,
resultsCount: benchmark.results[0].length,
metadata: {
threshold,
entityCount: 1000,
searchTerm: 'JavaScrpt'
}
});
console.log(` ā
Threshold ${threshold}: ${benchmark.avgTime.toFixed(2)}ms avg, ${benchmark.results[0].length} results`);
}
}
async runStrategyComparisonBenchmarks() {
console.log('\nāļø Running Strategy Comparison Benchmarks...');
const config = {
useDatabaseSearch: false,
threshold: 0.3,
clientSideFallback: true
};
const entities = PerformanceUtils.generateRealisticTestEntities(1000);
const mockPool = {};
const mockDb = {};
// PostgreSQL strategy
const pgStrategy = new PostgreSQLFuzzyStrategy(config, mockPool, 'test-project');
const pgBenchmark = await PerformanceUtils.runBenchmark(async () => {
return pgStrategy.searchClientSide(entities, 'Docker');
}, 20);
this.results.push({
category: 'Strategy Comparison',
test: 'PostgreSQL Strategy',
avgTime: pgBenchmark.avgTime,
minTime: pgBenchmark.minTime,
maxTime: pgBenchmark.maxTime,
medianTime: pgBenchmark.medianTime,
stdDev: pgBenchmark.stdDev,
iterations: 20,
resultsCount: pgBenchmark.results[0].length,
metadata: {
strategy: 'PostgreSQL',
entityCount: 1000,
searchTerm: 'Docker'
}
});
// SQLite strategy
const sqliteStrategy = new SQLiteFuzzyStrategy(config, mockDb, 'test-project');
const sqliteBenchmark = await PerformanceUtils.runBenchmark(async () => {
return sqliteStrategy.searchClientSide(entities, 'Docker');
}, 20);
this.results.push({
category: 'Strategy Comparison',
test: 'SQLite Strategy',
avgTime: sqliteBenchmark.avgTime,
minTime: sqliteBenchmark.minTime,
maxTime: sqliteBenchmark.maxTime,
medianTime: sqliteBenchmark.medianTime,
stdDev: sqliteBenchmark.stdDev,
iterations: 20,
resultsCount: sqliteBenchmark.results[0].length,
metadata: {
strategy: 'SQLite',
entityCount: 1000,
searchTerm: 'Docker'
}
});
console.log(` ā
PostgreSQL: ${pgBenchmark.avgTime.toFixed(2)}ms avg`);
console.log(` ā
SQLite: ${sqliteBenchmark.avgTime.toFixed(2)}ms avg`);
}
generateReport() {
const timestamp = new Date().toISOString();
const summary = this.generateSummary();
return {
timestamp,
summary,
results: this.results,
environment: {
nodeVersion: process.version,
platform: process.platform,
arch: process.arch,
memoryLimit: process.memoryUsage().rss
}
};
}
generateSummary() {
const categories = [...new Set(this.results.map(r => r.category))];
const categoryStats = {};
for (const category of categories) {
const categoryResults = this.results.filter(r => r.category === category);
const avgTimes = categoryResults.map(r => r.avgTime).filter(t => t > 0);
if (avgTimes.length > 0) {
categoryStats[category] = {
testCount: categoryResults.length,
avgTime: avgTimes.reduce((sum, time) => sum + time, 0) / avgTimes.length,
minTime: Math.min(...avgTimes),
maxTime: Math.max(...avgTimes)
};
}
else {
categoryStats[category] = {
testCount: categoryResults.length,
avgTime: 0,
minTime: 0,
maxTime: 0
};
}
}
return {
totalTests: this.results.length,
categories: categoryStats,
overallPerformance: this.assessOverallPerformance()
};
}
assessOverallPerformance() {
const searchResults = this.results.filter(r => r.category === 'Client-side Search');
const largeDatasetResult = searchResults.find(r => r.test.includes('Large'));
if (!largeDatasetResult)
return 'Unknown';
if (largeDatasetResult.avgTime < 1000)
return 'Excellent';
if (largeDatasetResult.avgTime < 2000)
return 'Good';
if (largeDatasetResult.avgTime < 5000)
return 'Acceptable';
return 'Needs Improvement';
}
async saveReport(report) {
const reportsDir = path.join(process.cwd(), 'benchmark-reports');
await fs.mkdir(reportsDir, { recursive: true });
const filename = `benchmark-report-${new Date().toISOString().replace(/[:.]/g, '-')}.json`;
const filepath = path.join(reportsDir, filename);
await fs.writeFile(filepath, JSON.stringify(report, null, 2));
console.log(`\nš Benchmark report saved to: ${filepath}`);
}
}
//# sourceMappingURL=benchmark-runner.js.map