claude-flow
Version:
Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration
480 lines • 20.1 kB
JavaScript
/**
* V3 CLI RuVector Benchmark Command
* Performance benchmarking for RuVector PostgreSQL Bridge
*/
import { output } from '../../output.js';
import { confirm } from '../../prompt.js';
/**
* Get PostgreSQL connection config from context
*/
function getConnectionConfig(ctx) {
return {
host: ctx.flags.host || process.env.PGHOST || 'localhost',
port: parseInt(ctx.flags.port || process.env.PGPORT || '5432', 10),
database: ctx.flags.database || process.env.PGDATABASE || '',
user: ctx.flags.user || process.env.PGUSER || 'postgres',
password: ctx.flags.password || process.env.PGPASSWORD || '',
ssl: ctx.flags.ssl || process.env.PGSSLMODE === 'require',
schema: ctx.flags.schema || 'claude_flow',
};
}
/**
* Generate random vector
*/
function generateRandomVector(dimensions) {
const vector = [];
for (let i = 0; i < dimensions; i++) {
vector.push(Math.random() * 2 - 1);
}
// Normalize
const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
return vector.map(v => v / magnitude);
}
/**
* Calculate percentile from sorted array
*/
function percentile(sortedArr, p) {
const index = Math.ceil((p / 100) * sortedArr.length) - 1;
return sortedArr[Math.max(0, index)];
}
/**
* RuVector benchmark command
*/
export const benchmarkCommand = {
name: 'benchmark',
description: 'Performance benchmarking',
options: [
{
name: 'vectors',
short: 'n',
description: 'Number of test vectors',
type: 'number',
default: 10000,
},
{
name: 'dimensions',
short: 'd',
description: 'Vector dimensions',
type: 'number',
default: 1536,
},
{
name: 'queries',
short: 'q',
description: 'Number of test queries',
type: 'number',
default: 100,
},
{
name: 'k',
description: 'Top-k results to retrieve',
type: 'number',
default: 10,
},
{
name: 'metric',
short: 'm',
description: 'Distance metric',
type: 'string',
default: 'cosine',
choices: ['cosine', 'l2', 'inner'],
},
{
name: 'index',
short: 'i',
description: 'Index type to test',
type: 'string',
default: 'hnsw',
choices: ['hnsw', 'ivfflat', 'none'],
},
{
name: 'batch-size',
description: 'Batch size for inserts',
type: 'number',
default: 1000,
},
{
name: 'cleanup',
description: 'Clean up test data after benchmark',
type: 'boolean',
default: true,
},
{
name: 'host',
short: 'h',
description: 'PostgreSQL host',
type: 'string',
default: 'localhost',
},
{
name: 'port',
short: 'p',
description: 'PostgreSQL port',
type: 'number',
default: 5432,
},
{
name: 'database',
description: 'Database name',
type: 'string',
},
{
name: 'user',
short: 'u',
description: 'Database user',
type: 'string',
},
{
name: 'password',
description: 'Database password',
type: 'string',
},
{
name: 'ssl',
description: 'Enable SSL',
type: 'boolean',
default: false,
},
{
name: 'schema',
short: 's',
description: 'Schema name',
type: 'string',
default: 'claude_flow',
},
],
examples: [
{ command: 'claude-flow ruvector benchmark', description: 'Run default benchmark' },
{ command: 'claude-flow ruvector benchmark --vectors 50000', description: 'Benchmark with 50k vectors' },
{ command: 'claude-flow ruvector benchmark --index ivfflat', description: 'Test IVFFlat index' },
{ command: 'claude-flow ruvector benchmark --dimensions 768 --metric l2', description: 'Custom dimensions and metric' },
],
action: async (ctx) => {
const config = getConnectionConfig(ctx);
const numVectors = parseInt(ctx.flags.vectors || '10000', 10);
const dimensions = parseInt(ctx.flags.dimensions || '1536', 10);
const numQueries = parseInt(ctx.flags.queries || '100', 10);
const topK = parseInt(ctx.flags.k || '10', 10);
const metric = ctx.flags.metric || 'cosine';
const indexType = ctx.flags.index || 'hnsw';
const batchSize = parseInt(ctx.flags['batch-size'] || '1000', 10);
const cleanup = ctx.flags.cleanup !== false;
output.writeln();
output.writeln(output.bold('RuVector Performance Benchmark'));
output.writeln(output.dim('='.repeat(60)));
output.writeln();
if (!config.database) {
output.printError('Database name is required. Use --database or -d flag, or set PGDATABASE env.');
return { success: false, exitCode: 1 };
}
// Show benchmark configuration
output.writeln(output.highlight('Benchmark Configuration:'));
output.printTable({
columns: [
{ key: 'setting', header: 'Setting', width: 20 },
{ key: 'value', header: 'Value', width: 20 },
],
data: [
{ setting: 'Vectors', value: numVectors.toLocaleString() },
{ setting: 'Dimensions', value: dimensions.toLocaleString() },
{ setting: 'Queries', value: numQueries.toLocaleString() },
{ setting: 'Top-K', value: topK.toLocaleString() },
{ setting: 'Metric', value: metric },
{ setting: 'Index Type', value: indexType.toUpperCase() },
{ setting: 'Batch Size', value: batchSize.toLocaleString() },
],
});
output.writeln();
// Confirm large benchmarks
if (numVectors >= 50000 && ctx.interactive) {
const confirmRun = await confirm({
message: `This will insert ${numVectors.toLocaleString()} vectors. Continue?`,
default: true,
});
if (!confirmRun) {
output.printInfo('Benchmark cancelled');
return { success: false, exitCode: 0 };
}
}
const spinner = output.createSpinner({ text: 'Connecting to PostgreSQL...', spinner: 'dots' });
spinner.start();
const results = {
config: { numVectors, dimensions, numQueries, topK, metric, indexType },
insert: {},
query: {},
memory: {},
};
try {
// Import pg
let pg = null;
try {
pg = await import('pg');
}
catch {
spinner.fail('PostgreSQL driver not found');
output.printError('Install pg package: npm install pg');
return { success: false, exitCode: 1 };
}
const client = new pg.Client({
host: config.host,
port: config.port,
database: config.database,
user: config.user,
password: config.password,
ssl: config.ssl ? { rejectUnauthorized: false } : false,
});
await client.connect();
spinner.succeed('Connected to PostgreSQL');
// Create benchmark table
const benchmarkTable = `${config.schema}.benchmark_${Date.now()}`;
spinner.setText('Creating benchmark table...');
spinner.start();
await client.query(`
CREATE TABLE ${benchmarkTable} (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
embedding vector(${dimensions}),
created_at TIMESTAMPTZ DEFAULT NOW()
)
`);
spinner.succeed('Benchmark table created');
// Insert vectors
spinner.setText(`Inserting ${numVectors.toLocaleString()} vectors...`);
spinner.start();
const insertStart = Date.now();
let insertedCount = 0;
for (let batch = 0; batch < Math.ceil(numVectors / batchSize); batch++) {
const batchStart = batch * batchSize;
const batchEnd = Math.min(batchStart + batchSize, numVectors);
const batchVectors = [];
for (let i = batchStart; i < batchEnd; i++) {
const vector = generateRandomVector(dimensions);
batchVectors.push(`('[${vector.join(',')}]')`);
}
await client.query(`
INSERT INTO ${benchmarkTable} (embedding)
VALUES ${batchVectors.join(',')}
`);
insertedCount = batchEnd;
spinner.setText(`Inserting vectors... ${insertedCount.toLocaleString()}/${numVectors.toLocaleString()}`);
}
const insertDuration = Date.now() - insertStart;
const insertThroughput = Math.round(numVectors / (insertDuration / 1000));
results.insert = {
totalTime: insertDuration,
throughput: insertThroughput,
vectorsInserted: numVectors,
};
spinner.succeed(`Inserted ${numVectors.toLocaleString()} vectors in ${(insertDuration / 1000).toFixed(2)}s (${insertThroughput.toLocaleString()} vectors/sec)`);
// Create index
if (indexType !== 'none') {
spinner.setText(`Creating ${indexType.toUpperCase()} index...`);
spinner.start();
const indexStart = Date.now();
const metricOp = metric === 'cosine' ? 'vector_cosine_ops' :
metric === 'l2' ? 'vector_l2_ops' : 'vector_ip_ops';
if (indexType === 'hnsw') {
await client.query(`
CREATE INDEX idx_benchmark_hnsw
ON ${benchmarkTable}
USING hnsw (embedding ${metricOp})
WITH (m = 16, ef_construction = 64)
`);
}
else if (indexType === 'ivfflat') {
// Need to train IVFFlat with existing data
const lists = Math.max(100, Math.floor(numVectors / 1000));
await client.query(`
CREATE INDEX idx_benchmark_ivfflat
ON ${benchmarkTable}
USING ivfflat (embedding ${metricOp})
WITH (lists = ${lists})
`);
}
const indexDuration = Date.now() - indexStart;
results.insert.indexTime = indexDuration;
spinner.succeed(`${indexType.toUpperCase()} index created in ${(indexDuration / 1000).toFixed(2)}s`);
// Set search parameters
if (indexType === 'hnsw') {
await client.query(`SET hnsw.ef_search = 100`);
}
else if (indexType === 'ivfflat') {
await client.query(`SET ivfflat.probes = 10`);
}
}
// Run queries
spinner.setText(`Running ${numQueries} queries...`);
spinner.start();
const queryLatencies = [];
const distanceOp = metric === 'cosine' ? '<=>' :
metric === 'l2' ? '<->' : '<#>';
for (let q = 0; q < numQueries; q++) {
const queryVector = generateRandomVector(dimensions);
const queryStart = Date.now();
await client.query(`
SELECT id, embedding ${distanceOp} '[${queryVector.join(',')}]' as distance
FROM ${benchmarkTable}
ORDER BY embedding ${distanceOp} '[${queryVector.join(',')}]'
LIMIT ${topK}
`);
queryLatencies.push(Date.now() - queryStart);
if (q % 10 === 0) {
spinner.setText(`Running queries... ${q + 1}/${numQueries}`);
}
}
// Calculate query statistics
queryLatencies.sort((a, b) => a - b);
const avgLatency = queryLatencies.reduce((a, b) => a + b, 0) / queryLatencies.length;
const p50 = percentile(queryLatencies, 50);
const p95 = percentile(queryLatencies, 95);
const p99 = percentile(queryLatencies, 99);
const minLatency = queryLatencies[0];
const maxLatency = queryLatencies[queryLatencies.length - 1];
const qps = Math.round(1000 / avgLatency);
results.query = {
totalQueries: numQueries,
avgLatency,
p50,
p95,
p99,
minLatency,
maxLatency,
qps,
};
spinner.succeed(`Completed ${numQueries} queries`);
// Get memory usage
spinner.setText('Analyzing memory usage...');
spinner.start();
const sizeResult = await client.query(`
SELECT
pg_relation_size('${benchmarkTable}') as table_size,
pg_total_relation_size('${benchmarkTable}') as total_size,
pg_indexes_size('${benchmarkTable}') as index_size
`);
const tableSize = parseInt(sizeResult.rows[0].table_size, 10);
const totalSize = parseInt(sizeResult.rows[0].total_size, 10);
const indexSize = parseInt(sizeResult.rows[0].index_size, 10);
const bytesPerVector = totalSize / numVectors;
results.memory = {
tableSize,
indexSize,
totalSize,
bytesPerVector,
vectorDimensions: dimensions,
};
spinner.succeed('Memory analysis complete');
// Calculate recall (if we have ground truth)
// For now, we'll estimate based on index type
let estimatedRecall = 1.0;
if (indexType === 'hnsw') {
estimatedRecall = 0.99; // HNSW typically achieves 99%+ recall with default params
}
else if (indexType === 'ivfflat') {
estimatedRecall = 0.95; // IVFFlat typically 95% with probes=10
}
results.query.estimatedRecall = estimatedRecall;
// Cleanup
if (cleanup) {
spinner.setText('Cleaning up benchmark data...');
spinner.start();
await client.query(`DROP TABLE IF EXISTS ${benchmarkTable}`);
spinner.succeed('Benchmark data cleaned up');
}
else {
output.printInfo(`Benchmark table retained: ${benchmarkTable}`);
}
await client.end();
// Display results
output.writeln();
output.writeln(output.bold('Benchmark Results'));
output.writeln(output.dim('-'.repeat(60)));
output.writeln();
// Insert performance
output.writeln(output.highlight('Insert Performance:'));
output.printTable({
columns: [
{ key: 'metric', header: 'Metric', width: 25 },
{ key: 'value', header: 'Value', width: 25 },
],
data: [
{ metric: 'Total Vectors', value: numVectors.toLocaleString() },
{ metric: 'Total Time', value: `${(insertDuration / 1000).toFixed(2)}s` },
{ metric: 'Throughput', value: `${insertThroughput.toLocaleString()} vectors/sec` },
{ metric: 'Index Build Time', value: results.insert.indexTime
? `${(results.insert.indexTime / 1000).toFixed(2)}s`
: 'N/A' },
],
});
output.writeln();
// Query performance
output.writeln(output.highlight('Query Performance:'));
output.printTable({
columns: [
{ key: 'metric', header: 'Metric', width: 25 },
{ key: 'value', header: 'Value', width: 25 },
],
data: [
{ metric: 'Total Queries', value: numQueries.toLocaleString() },
{ metric: 'Avg Latency', value: `${avgLatency.toFixed(2)}ms` },
{ metric: 'P50 Latency', value: `${p50.toFixed(2)}ms` },
{ metric: 'P95 Latency', value: `${p95.toFixed(2)}ms` },
{ metric: 'P99 Latency', value: `${p99.toFixed(2)}ms` },
{ metric: 'Min/Max Latency', value: `${minLatency.toFixed(2)}ms / ${maxLatency.toFixed(2)}ms` },
{ metric: 'Queries/Second', value: qps.toLocaleString() },
{ metric: 'Estimated Recall@K', value: `${(estimatedRecall * 100).toFixed(1)}%` },
],
});
output.writeln();
// Memory usage
output.writeln(output.highlight('Memory Usage:'));
const formatBytes = (b) => {
if (b < 1024)
return `${b} B`;
if (b < 1024 * 1024)
return `${(b / 1024).toFixed(2)} KB`;
if (b < 1024 * 1024 * 1024)
return `${(b / 1024 / 1024).toFixed(2)} MB`;
return `${(b / 1024 / 1024 / 1024).toFixed(2)} GB`;
};
output.printTable({
columns: [
{ key: 'metric', header: 'Metric', width: 25 },
{ key: 'value', header: 'Value', width: 25 },
],
data: [
{ metric: 'Table Size', value: formatBytes(tableSize) },
{ metric: 'Index Size', value: formatBytes(indexSize) },
{ metric: 'Total Size', value: formatBytes(totalSize) },
{ metric: 'Bytes per Vector', value: `${bytesPerVector.toFixed(2)} bytes` },
],
});
output.writeln();
// Summary box
const grade = qps >= 1000 ? 'Excellent' :
qps >= 500 ? 'Good' :
qps >= 100 ? 'Fair' : 'Needs Optimization';
const gradeColor = qps >= 1000 ? output.success.bind(output) :
qps >= 500 ? output.highlight.bind(output) :
qps >= 100 ? output.warning.bind(output) : output.error.bind(output);
output.printBox([
`Performance Grade: ${gradeColor(grade)}`,
'',
`Throughput: ${insertThroughput.toLocaleString()} inserts/sec, ${qps.toLocaleString()} queries/sec`,
`Latency: ${avgLatency.toFixed(2)}ms avg, ${p99.toFixed(2)}ms p99`,
`Memory: ${formatBytes(bytesPerVector)} per ${dimensions}-dim vector`,
`Recall: ~${(estimatedRecall * 100).toFixed(0)}% @ k=${topK}`,
'',
indexType === 'hnsw' ? 'HNSW index provides excellent recall with good performance.' :
indexType === 'ivfflat' ? 'IVFFlat index balances memory usage and query speed.' :
'No index: exact search provides 100% recall but slower queries.',
].join('\n'), 'Summary');
return { success: true, data: results };
}
catch (error) {
spinner.fail('Benchmark failed');
output.printError(error instanceof Error ? error.message : String(error));
return { success: false, exitCode: 1 };
}
},
};
export default benchmarkCommand;
//# sourceMappingURL=benchmark.js.map