UNPKG

@re-shell/cli

Version:

Full-stack development platform uniting microservices and microfrontends. Build complete applications with .NET (ASP.NET Core Web API, Minimal API), Java (Spring Boot, Quarkus, Micronaut, Vert.x), Rust (Actix-Web, Warp, Rocket, Axum), Python (FastAPI, Dja

1,587 lines (1,314 loc) 62.8 kB
"use strict"; /** * C++ Google Benchmark Integration Generator * Generates Google Benchmark configuration and templates for C++ projects */ Object.defineProperty(exports, "__esModule", { value: true }); exports.CppBenchmarkGenerator = void 0; class CppBenchmarkGenerator { static generate(config) { const { projectName, benchmarkVersion = "1.8.3", enableMemoryCounters = true, enableTimeUnit = 'us', enableMultithreading = true, enableJsonOutput = true, enableConsoleColors = true } = config; return { 'benchmarks/CMakeLists.txt': this.generateBenchmarkCMake(projectName, benchmarkVersion), 'benchmarks/main_benchmark.cpp': this.generateMainBenchmark(projectName), 'benchmarks/api_benchmark.cpp': this.generateApiBenchmark(projectName), 'benchmarks/performance_benchmark.cpp': this.generatePerformanceBenchmark(projectName), 'scripts/run_benchmarks.sh': this.generateBenchmarkScript(enableJsonOutput, enableConsoleColors), 'scripts/benchmark_analysis.py': this.generateBenchmarkAnalysis(), 'benchmarks/README.md': this.generateBenchmarkReadme(projectName), '.github/workflows/benchmark.yml': this.generateBenchmarkCI(projectName), 'cmake/benchmark.cmake': this.generateBenchmarkCMakeModule(), 'benchmarks/utils/benchmark_utils.hpp': this.generateBenchmarkUtils(), 'benchmarks/utils/benchmark_utils.cpp': this.generateBenchmarkUtilsImpl() }; } static generateBenchmarkCMake(projectName, version) { return `# Google Benchmark Configuration for ${projectName} cmake_minimum_required(VERSION 3.16) # Find or download Google Benchmark find_package(benchmark QUIET) if(NOT benchmark_FOUND) include(FetchContent) FetchContent_Declare( googlebenchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v${version} ) # Configure benchmark options set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing") set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable gtest in benchmark") set(BENCHMARK_ENABLE_ASSEMBLY_TESTS OFF CACHE BOOL "Disable assembly tests") FetchContent_MakeAvailable(googlebenchmark) endif() # Create benchmark utilities library add_library(benchmark_utils utils/benchmark_utils.cpp utils/benchmark_utils.hpp ) target_link_libraries(benchmark_utils PUBLIC benchmark::benchmark) target_include_directories(benchmark_utils PUBLIC utils) # Main benchmark executable add_executable(${projectName}_benchmark main_benchmark.cpp api_benchmark.cpp performance_benchmark.cpp ) target_link_libraries(${projectName}_benchmark PRIVATE benchmark::benchmark benchmark_utils \${PROJECT_NAME}_lib # Link to your main library ) # Compiler optimizations for benchmarks target_compile_options(${projectName}_benchmark PRIVATE \$<\$<CXX_COMPILER_ID:GNU>:-O3 -march=native -mtune=native> \$<\$<CXX_COMPILER_ID:Clang>:-O3 -march=native -mtune=native> \$<\$<CXX_COMPILER_ID:MSVC>:/O2> ) # Benchmark registration and discovery target_compile_definitions(${projectName}_benchmark PRIVATE BENCHMARK_ENABLE_MEMORY_COUNTERS=1 BENCHMARK_ENABLE_THREADING=1 ) # Custom benchmark targets add_custom_target(run_benchmarks COMMAND \${CMAKE_CURRENT_BINARY_DIR}/${projectName}_benchmark DEPENDS ${projectName}_benchmark COMMENT "Running performance benchmarks" ) add_custom_target(benchmark_json COMMAND \${CMAKE_CURRENT_BINARY_DIR}/${projectName}_benchmark --benchmark_format=json --benchmark_out=benchmark_results.json DEPENDS ${projectName}_benchmark COMMENT "Running benchmarks with JSON output" ) add_custom_target(benchmark_detailed COMMAND \${CMAKE_CURRENT_BINARY_DIR}/${projectName}_benchmark --benchmark_repetitions=10 --benchmark_display_aggregates_only=true DEPENDS ${projectName}_benchmark COMMENT "Running detailed benchmark analysis" )`; } static generateMainBenchmark(projectName) { return `/** * Main Benchmark Suite for ${projectName} * Comprehensive performance testing and analysis */ #include <benchmark/benchmark.h> #include "benchmark_utils.hpp" #include <vector> #include <string> #include <memory> #include <chrono> #include <random> // Example: String processing benchmark static void BM_StringCreation(benchmark::State& state) { for (auto _ : state) { std::string empty_string; benchmark::DoNotOptimize(empty_string); } } BENCHMARK(BM_StringCreation); // Example: String concatenation benchmark static void BM_StringCopy(benchmark::State& state) { std::string x = "hello world"; for (auto _ : state) { std::string copy(x); benchmark::DoNotOptimize(copy); } } BENCHMARK(BM_StringCopy); // Memory allocation benchmark static void BM_VectorCreation(benchmark::State& state) { for (auto _ : state) { std::vector<int> v; v.reserve(state.range(0)); for (int i = 0; i < state.range(0); ++i) { v.push_back(i); } benchmark::DoNotOptimize(v.data()); benchmark::ClobberMemory(); } } BENCHMARK(BM_VectorCreation)->Range(8, 8<<10); // Memory access patterns static void BM_SequentialAccess(benchmark::State& state) { std::vector<int> data(state.range(0)); std::iota(data.begin(), data.end(), 0); for (auto _ : state) { long sum = 0; for (size_t i = 0; i < data.size(); ++i) { sum += data[i]; } benchmark::DoNotOptimize(sum); } state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int)); } BENCHMARK(BM_SequentialAccess)->Range(1024, 1024*1024); // Random access benchmark static void BM_RandomAccess(benchmark::State& state) { std::vector<int> data(state.range(0)); std::iota(data.begin(), data.end(), 0); std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<size_t> dis(0, data.size() - 1); for (auto _ : state) { long sum = 0; for (int i = 0; i < 1000; ++i) { sum += data[dis(gen)]; } benchmark::DoNotOptimize(sum); } } BENCHMARK(BM_RandomAccess)->Range(1024, 1024*1024); // Multithreaded benchmark example static void BM_MultiThreadedWork(benchmark::State& state) { if (state.thread_index == 0) { // Setup shared data } for (auto _ : state) { // Work that each thread performs std::vector<int> local_data(1000); std::iota(local_data.begin(), local_data.end(), state.thread_index * 1000); long sum = 0; for (int val : local_data) { sum += val * val; } benchmark::DoNotOptimize(sum); } } BENCHMARK(BM_MultiThreadedWork)->Threads(1)->Threads(2)->Threads(4)->Threads(8); BENCHMARK_MAIN();`; } static generateApiBenchmark(projectName) { return `/** * API Performance Benchmarks for ${projectName} * HTTP request/response processing benchmarks */ #include <benchmark/benchmark.h> #include "benchmark_utils.hpp" #include <string> #include <vector> #include <memory> // Mock API endpoint processing static void BM_JsonParsing(benchmark::State& state) { const std::string json_data = R"({ "id": 12345, "name": "Test User", "email": "test@example.com", "metadata": { "preferences": ["setting1", "setting2", "setting3"], "scores": [85, 92, 78, 95, 87] } })"; for (auto _ : state) { // Simulate JSON parsing BenchmarkUtils::ProcessJsonData(json_data); benchmark::ClobberMemory(); } state.SetBytesProcessed(state.iterations() * json_data.size()); } BENCHMARK(BM_JsonParsing); // Database query simulation static void BM_DatabaseQuery(benchmark::State& state) { auto connection = BenchmarkUtils::CreateMockConnection(); for (auto _ : state) { auto result = BenchmarkUtils::ExecuteMockQuery( connection, "SELECT * FROM users WHERE id = ?", {std::to_string(state.range(0))} ); benchmark::DoNotOptimize(result); } } BENCHMARK(BM_DatabaseQuery)->Range(1, 10000); // HTTP request processing static void BM_RequestProcessing(benchmark::State& state) { for (auto _ : state) { auto request = BenchmarkUtils::CreateMockRequest("GET", "/api/users/123"); auto response = BenchmarkUtils::ProcessRequest(request); benchmark::DoNotOptimize(response); } } BENCHMARK(BM_RequestProcessing); // Concurrent request handling static void BM_ConcurrentRequests(benchmark::State& state) { if (state.thread_index == 0) { BenchmarkUtils::InitializeServer(); } for (auto _ : state) { auto request = BenchmarkUtils::CreateMockRequest( "POST", "/api/data", BenchmarkUtils::GenerateTestPayload(state.range(0)) ); auto response = BenchmarkUtils::ProcessRequestConcurrent(request); benchmark::DoNotOptimize(response); } if (state.thread_index == 0) { BenchmarkUtils::ShutdownServer(); } } BENCHMARK(BM_ConcurrentRequests) ->Range(1024, 8<<10) ->Threads(1)->Threads(4)->Threads(8)->Threads(16); // Authentication/Authorization benchmark static void BM_Authentication(benchmark::State& state) { auto auth_service = BenchmarkUtils::CreateAuthService(); for (auto _ : state) { std::string token = BenchmarkUtils::GenerateJWT(); bool is_valid = BenchmarkUtils::ValidateToken(auth_service, token); benchmark::DoNotOptimize(is_valid); } } BENCHMARK(BM_Authentication); // Data serialization benchmark static void BM_DataSerialization(benchmark::State& state) { auto test_objects = BenchmarkUtils::GenerateTestObjects(state.range(0)); for (auto _ : state) { std::string serialized = BenchmarkUtils::SerializeToJson(test_objects); benchmark::DoNotOptimize(serialized.data()); } state.SetItemsProcessed(state.iterations() * state.range(0)); } BENCHMARK(BM_DataSerialization)->Range(1, 1000); // Data deserialization benchmark static void BM_DataDeserialization(benchmark::State& state) { auto test_data = BenchmarkUtils::GenerateTestJsonArray(state.range(0)); for (auto _ : state) { auto objects = BenchmarkUtils::DeserializeFromJson(test_data); benchmark::DoNotOptimize(objects.data()); } state.SetItemsProcessed(state.iterations() * state.range(0)); } BENCHMARK(BM_DataDeserialization)->Range(1, 1000);`; } static generatePerformanceBenchmark(projectName) { return `/** * Performance Critical Path Benchmarks for ${projectName} * Core algorithm and data structure benchmarks */ #include <benchmark/benchmark.h> #include "benchmark_utils.hpp" #include <algorithm> #include <numeric> #include <vector> #include <unordered_map> #include <map> #include <set> #include <string> #include <memory> // Sorting algorithms comparison static void BM_StdSort(benchmark::State& state) { for (auto _ : state) { state.PauseTiming(); auto data = BenchmarkUtils::GenerateRandomInts(state.range(0)); state.ResumeTiming(); std::sort(data.begin(), data.end()); benchmark::DoNotOptimize(data.data()); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_StdSort)->Range(1<<10, 1<<20)->Complexity(); // Hash map vs tree map performance static void BM_UnorderedMapInsert(benchmark::State& state) { std::unordered_map<int, std::string> map; auto keys = BenchmarkUtils::GenerateRandomInts(state.range(0)); for (auto _ : state) { for (int key : keys) { map[key] = "value_" + std::to_string(key); } benchmark::ClobberMemory(); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_UnorderedMapInsert)->Range(1<<8, 1<<16)->Complexity(); static void BM_MapInsert(benchmark::State& state) { std::map<int, std::string> map; auto keys = BenchmarkUtils::GenerateRandomInts(state.range(0)); for (auto _ : state) { for (int key : keys) { map[key] = "value_" + std::to_string(key); } benchmark::ClobberMemory(); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_MapInsert)->Range(1<<8, 1<<16)->Complexity(); // String processing benchmarks static void BM_StringConcatenation(benchmark::State& state) { auto strings = BenchmarkUtils::GenerateRandomStrings(state.range(0), 20); for (auto _ : state) { std::string result; for (const auto& str : strings) { result += str; } benchmark::DoNotOptimize(result.data()); } state.SetItemsProcessed(state.iterations() * state.range(0)); } BENCHMARK(BM_StringConcatenation)->Range(1, 1000); static void BM_StringJoin(benchmark::State& state) { auto strings = BenchmarkUtils::GenerateRandomStrings(state.range(0), 20); for (auto _ : state) { std::string result = BenchmarkUtils::JoinStrings(strings, ","); benchmark::DoNotOptimize(result.data()); } state.SetItemsProcessed(state.iterations() * state.range(0)); } BENCHMARK(BM_StringJoin)->Range(1, 1000); // Memory allocation patterns static void BM_StackAllocation(benchmark::State& state) { for (auto _ : state) { int data[1000]; std::iota(data, data + 1000, 0); benchmark::DoNotOptimize(data); } } BENCHMARK(BM_StackAllocation); static void BM_HeapAllocation(benchmark::State& state) { for (auto _ : state) { auto data = std::make_unique<int[]>(1000); std::iota(data.get(), data.get() + 1000, 0); benchmark::DoNotOptimize(data.get()); } } BENCHMARK(BM_HeapAllocation); // Cache efficiency benchmarks static void BM_CacheFriendlyAccess(benchmark::State& state) { std::vector<int> data(state.range(0)); std::iota(data.begin(), data.end(), 0); for (auto _ : state) { long sum = 0; // Sequential access - cache friendly for (size_t i = 0; i < data.size(); ++i) { sum += data[i]; } benchmark::DoNotOptimize(sum); } state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int)); } BENCHMARK(BM_CacheFriendlyAccess)->Range(1<<10, 1<<22); static void BM_CacheUnfriendlyAccess(benchmark::State& state) { std::vector<int> data(state.range(0)); std::iota(data.begin(), data.end(), 0); for (auto _ : state) { long sum = 0; // Stride access - cache unfriendly size_t stride = 64; // Assume 64-byte cache lines for (size_t i = 0; i < data.size(); i += stride) { sum += data[i]; } benchmark::DoNotOptimize(sum); } state.SetBytesProcessed(state.iterations() * (state.range(0) / 64) * sizeof(int)); } BENCHMARK(BM_CacheUnfriendlyAccess)->Range(1<<10, 1<<22); // Algorithm complexity verification static void BM_LinearSearch(benchmark::State& state) { auto data = BenchmarkUtils::GenerateRandomInts(state.range(0)); int target = data[data.size() / 2]; for (auto _ : state) { auto it = std::find(data.begin(), data.end(), target); benchmark::DoNotOptimize(it); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_LinearSearch)->Range(1<<8, 1<<16)->Complexity(); static void BM_BinarySearch(benchmark::State& state) { auto data = BenchmarkUtils::GenerateRandomInts(state.range(0)); std::sort(data.begin(), data.end()); int target = data[data.size() / 2]; for (auto _ : state) { bool found = std::binary_search(data.begin(), data.end(), target); benchmark::DoNotOptimize(found); } state.SetComplexityN(state.range(0)); } BENCHMARK(BM_BinarySearch)->Range(1<<8, 1<<16)->Complexity();`; } static generateBenchmarkScript(enableJsonOutput, enableConsoleColors) { return `#!/bin/bash # Google Benchmark Runner Script # Comprehensive performance testing and analysis set -euo pipefail # Configuration BENCHMARK_BINARY="./build/benchmarks/\\$(basename \\$(pwd))_benchmark" RESULTS_DIR="benchmark_results" TIMESTAMP="\\$(date +%Y%m%d_%H%M%S)" # Create results directory mkdir -p "\\${RESULTS_DIR}" # Colors for output if [[ "${enableConsoleColors}" == "true" ]]; then RED='\\033[0;31m' GREEN='\\033[0;32m' BLUE='\\033[0;34m' YELLOW='\\033[1;33m' NC='\\033[0m' # No Color else RED='' GREEN='' BLUE='' YELLOW='' NC='' fi echo -e "\\${BLUE}=== Performance Benchmark Suite ===\\${NC}" # Check if benchmark binary exists if [[ ! -f "\\${BENCHMARK_BINARY}" ]]; then echo -e "\\${RED}Error: Benchmark binary not found at \\${BENCHMARK_BINARY}\\${NC}" echo "Please build the project first: cmake --build build --target \\$(basename \\$(pwd))_benchmark" exit 1 fi # Function to run benchmark with specific options run_benchmark() { local name="\\$1" local args="\\$2" local output_file="\\${RESULTS_DIR}/\\${name}_\\${TIMESTAMP}" echo -e "\\${YELLOW}Running \\${name} benchmark...\\${NC}" if [[ "${enableJsonOutput}" == "true" ]]; then \\${BENCHMARK_BINARY} \\${args} \\ --benchmark_format=json \\ --benchmark_out="\\${output_file}.json" \\ --benchmark_counters_tabular=true \\ | tee "\\${output_file}.txt" else \\${BENCHMARK_BINARY} \\${args} | tee "\\${output_file}.txt" fi echo -e "\\${GREEN}Results saved to \\${output_file}.*\\${NC}" echo } # Quick benchmark run run_benchmark "quick" "--benchmark_filter=BM_.*" # Detailed benchmark with repetitions run_benchmark "detailed" "--benchmark_repetitions=5 --benchmark_display_aggregates_only=true" # Memory intensive benchmarks run_benchmark "memory" "--benchmark_filter=.*Memory.* --benchmark_memory_counters=true" # Multithreaded benchmarks run_benchmark "multithreaded" "--benchmark_filter=.*MultiThreaded.*" # Performance regression detection if [[ -f "\\${RESULTS_DIR}/baseline.json" ]]; then echo -e "\\${YELLOW}Running regression detection...\\${NC}" \\${BENCHMARK_BINARY} \\ --benchmark_format=json \\ --benchmark_out="\\${RESULTS_DIR}/current_\\${TIMESTAMP}.json" \\ --benchmark_filter=BM_.* # Compare with baseline (requires compare.py tool) if command -v python3 &> /dev/null; then python3 scripts/benchmark_analysis.py \\ "\\${RESULTS_DIR}/baseline.json" \\ "\\${RESULTS_DIR}/current_\\${TIMESTAMP}.json" \\ > "\\${RESULTS_DIR}/regression_report_\\${TIMESTAMP}.txt" echo -e "\\${GREEN}Regression report saved to \\${RESULTS_DIR}/regression_report_\\${TIMESTAMP}.txt\\${NC}" fi fi # Generate summary report echo -e "\\${BLUE}=== Benchmark Summary ===\\${NC}" echo "Timestamp: \\${TIMESTAMP}" echo "Results directory: \\${RESULTS_DIR}" echo "Binary: \\${BENCHMARK_BINARY}" if [[ "${enableJsonOutput}" == "true" ]]; then echo "JSON results available for analysis" echo "Use: python3 scripts/benchmark_analysis.py <json_file>" fi echo -e "\${GREEN}Benchmark suite completed successfully!\${NC}" # Optional: Upload results to performance tracking system if [[ -n "\${BENCHMARK_UPLOAD_URL:-}" ]]; then echo -e "\${YELLOW}Uploading results to performance tracking system...\${NC}" curl -X POST "\${BENCHMARK_UPLOAD_URL}" \\ -H "Content-Type: application/json" \\ -d @"\${RESULTS_DIR}/detailed_\${TIMESTAMP}.json" || true fi`; } static generateBenchmarkAnalysis() { return `#!/usr/bin/env python3 """ Google Benchmark Analysis Tool Analyzes benchmark results and generates performance reports """ import json import sys import argparse import statistics from typing import Dict, List, Any, Optional from datetime import datetime class BenchmarkAnalyzer: def __init__(self): self.thresholds = { 'regression': 1.10, # 10% slowdown 'improvement': 0.90, # 10% speedup 'significant_change': 0.05 # 5% change threshold } def load_benchmark_results(self, filename: str) -> Dict[str, Any]: """Load benchmark results from JSON file.""" try: with open(filename, 'r') as f: return json.load(f) except FileNotFoundError: print(f"Error: File {filename} not found") sys.exit(1) except json.JSONDecodeError as e: print(f"Error parsing JSON: {e}") sys.exit(1) def extract_benchmark_data(self, results: Dict[str, Any]) -> Dict[str, Dict[str, float]]: """Extract relevant benchmark data.""" benchmarks = {} for benchmark in results.get('benchmarks', []): name = benchmark['name'] benchmarks[name] = { 'cpu_time': benchmark['cpu_time'], 'real_time': benchmark['real_time'], 'iterations': benchmark['iterations'], 'bytes_per_second': benchmark.get('bytes_per_second', 0), 'items_per_second': benchmark.get('items_per_second', 0) } return benchmarks def compare_benchmarks(self, baseline: Dict[str, Dict[str, float]], current: Dict[str, Dict[str, float]]) -> Dict[str, Dict[str, Any]]: """Compare current results with baseline.""" comparison = {} for name in current: if name not in baseline: comparison[name] = { 'status': 'NEW', 'current': current[name], 'change': None } continue baseline_time = baseline[name]['cpu_time'] current_time = current[name]['cpu_time'] ratio = current_time / baseline_time change_pct = (ratio - 1.0) * 100 status = 'STABLE' if ratio > self.thresholds['regression']: status = 'REGRESSION' elif ratio < self.thresholds['improvement']: status = 'IMPROVEMENT' elif abs(change_pct) > self.thresholds['significant_change'] * 100: status = 'CHANGED' comparison[name] = { 'status': status, 'baseline': baseline[name], 'current': current[name], 'ratio': ratio, 'change_pct': change_pct } return comparison def generate_report(self, comparison: Dict[str, Dict[str, Any]]) -> str: """Generate a human-readable performance report.""" report_lines = [ "# Benchmark Performance Report", f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", "", "## Summary", "" ] status_counts = {} for data in comparison.values(): status = data['status'] status_counts[status] = status_counts.get(status, 0) + 1 for status, count in sorted(status_counts.items()): report_lines.append(f"- {status}: {count} benchmarks") report_lines.extend(["", "## Detailed Results", ""]) # Sort by change percentage (worst regressions first) sorted_benchmarks = sorted( comparison.items(), key=lambda x: x[1].get('change_pct', 0), reverse=True ) for name, data in sorted_benchmarks: status = data['status'] if status == 'NEW': current_time = data['current']['cpu_time'] report_lines.append(f"### {name} [NEW]") report_lines.append(f"- CPU Time: {current_time:.2f} ns") else: change_pct = data['change_pct'] current_time = data['current']['cpu_time'] baseline_time = data['baseline']['cpu_time'] emoji = "🔴" if status == 'REGRESSION' else "🟢" if status == 'IMPROVEMENT' else "🟡" report_lines.append(f"### {name} [{status}] {emoji}") report_lines.append(f"- Change: {change_pct:+.2f}%") report_lines.append(f"- Current: {current_time:.2f} ns") report_lines.append(f"- Baseline: {baseline_time:.2f} ns") report_lines.append("") return "\\n".join(report_lines) def analyze_single_file(self, filename: str) -> str: """Analyze a single benchmark file.""" results = self.load_benchmark_results(filename) benchmarks = self.extract_benchmark_data(results) report_lines = [ f"# Benchmark Analysis: {filename}", f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", "", "## Performance Summary", "" ] # Calculate statistics cpu_times = [b['cpu_time'] for b in benchmarks.values()] if cpu_times: report_lines.extend([ f"- Total benchmarks: {len(benchmarks)}", f"- Average CPU time: {statistics.mean(cpu_times):.2f} ns", f"- Median CPU time: {statistics.median(cpu_times):.2f} ns", f"- Min CPU time: {min(cpu_times):.2f} ns", f"- Max CPU time: {max(cpu_times):.2f} ns", "" ]) report_lines.extend(["## Benchmark Details", ""]) for name, data in sorted(benchmarks.items()): report_lines.append(f"### {name}") report_lines.append(f"- CPU Time: {data['cpu_time']:.2f} ns") report_lines.append(f"- Real Time: {data['real_time']:.2f} ns") report_lines.append(f"- Iterations: {data['iterations']:,}") if data['bytes_per_second'] > 0: report_lines.append(f"- Throughput: {data['bytes_per_second']/1e6:.2f} MB/s") if data['items_per_second'] > 0: report_lines.append(f"- Items/sec: {data['items_per_second']:,.0f}") report_lines.append("") return "\\n".join(report_lines) def main(): parser = argparse.ArgumentParser(description='Analyze Google Benchmark results') parser.add_argument('baseline', help='Baseline benchmark results (JSON)') parser.add_argument('current', nargs='?', help='Current benchmark results (JSON)') parser.add_argument('--output', '-o', help='Output file for report') args = parser.parse_args() analyzer = BenchmarkAnalyzer() if args.current: # Compare two files baseline_results = analyzer.load_benchmark_results(args.baseline) current_results = analyzer.load_benchmark_results(args.current) baseline_data = analyzer.extract_benchmark_data(baseline_results) current_data = analyzer.extract_benchmark_data(current_results) comparison = analyzer.compare_benchmarks(baseline_data, current_data) report = analyzer.generate_report(comparison) else: # Analyze single file report = analyzer.analyze_single_file(args.baseline) if args.output: with open(args.output, 'w') as f: f.write(report) print(f"Report saved to {args.output}") else: print(report) if __name__ == '__main__': main()`; } static generateBenchmarkReadme(projectName) { return `# ${projectName} Performance Benchmarks This directory contains comprehensive performance benchmarks using Google Benchmark framework. ## Overview The benchmark suite includes: - **Main Benchmarks** (\`main_benchmark.cpp\`): Core algorithm and data structure performance - **API Benchmarks** (\`api_benchmark.cpp\`): HTTP request/response processing - **Performance Benchmarks** (\`performance_benchmark.cpp\`): Critical path optimization ## Quick Start ### Building Benchmarks \`\`\`bash # Configure with benchmarks enabled cmake -B build -DCMAKE_BUILD_TYPE=Release # Build benchmark executable cmake --build build --target ${projectName}_benchmark \`\`\` ### Running Benchmarks \`\`\`bash # Quick benchmark run ./scripts/run_benchmarks.sh # Manual execution ./build/benchmarks/${projectName}_benchmark # With specific options ./build/benchmarks/${projectName}_benchmark --benchmark_filter=BM_String.* \`\`\` ## Benchmark Categories ### Memory and Data Structures - Vector creation and access patterns - Hash map vs tree map performance - Memory allocation strategies - Cache efficiency analysis ### String Processing - String creation and copying - Concatenation vs joining - Pattern matching and parsing ### API Performance - JSON parsing and serialization - Database query simulation - HTTP request processing - Authentication and authorization ### Concurrency - Multithreaded workload distribution - Concurrent request handling - Lock contention analysis ### Algorithm Complexity - Sorting algorithm comparisons - Search algorithm verification - Complexity analysis and validation ## Understanding Results ### Key Metrics - **CPU Time**: Time spent on CPU (excludes I/O wait) - **Real Time**: Wall clock time (includes everything) - **Iterations**: Number of benchmark iterations - **Bytes/sec**: Throughput for data processing - **Items/sec**: Processing rate for discrete items ### Performance Analysis \`\`\`bash # Generate detailed analysis python3 scripts/benchmark_analysis.py benchmark_results/detailed_TIMESTAMP.json # Compare with baseline python3 scripts/benchmark_analysis.py baseline.json current.json \`\`\` ### Result Interpretation - **Green (🟢)**: Performance improvement (>10% faster) - **Yellow (🟡)**: Stable performance (±5% change) - **Red (🔴)**: Performance regression (>10% slower) ## Benchmark Best Practices ### Writing Benchmarks 1. **Use DoNotOptimize()**: Prevent compiler optimization 2. **Use ClobberMemory()**: Prevent memory optimization 3. **Set proper ranges**: Test realistic data sizes 4. **Consider cache effects**: Test both hot and cold scenarios ### Example Benchmark \`\`\`cpp static void BM_YourFunction(benchmark::State& state) { // Setup (not measured) std::vector<int> data(state.range(0)); for (auto _ : state) { // Code being measured auto result = your_function(data); benchmark::DoNotOptimize(result); } // Optional metrics state.SetBytesProcessed(state.iterations() * state.range(0) * sizeof(int)); state.SetComplexityN(state.range(0)); } BENCHMARK(BM_YourFunction)->Range(1<<10, 1<<20)->Complexity(); \`\`\` ## Continuous Integration Benchmarks run automatically on: - Pull requests (regression detection) - Main branch commits (performance tracking) - Nightly builds (comprehensive analysis) ## Performance Regression Detection The CI system detects: - **Major regressions**: >20% performance loss - **Minor regressions**: 10-20% performance loss - **Significant changes**: >5% performance change ## Configuration ### Environment Variables - \`BENCHMARK_UPLOAD_URL\`: Upload results to tracking system - \`BENCHMARK_BASELINE\`: Path to baseline results file - \`BENCHMARK_REPETITIONS\`: Number of repetitions (default: 3) ### CMake Options - \`BENCHMARK_ENABLE_TESTING\`: Enable Google Benchmark tests - \`BENCHMARK_ENABLE_GTEST_TESTS\`: Enable Google Test integration - \`BENCHMARK_ENABLE_ASSEMBLY_TESTS\`: Enable assembly tests ## Troubleshooting ### Common Issues 1. **Inconsistent results**: Ensure consistent CPU frequency, disable turbo boost 2. **Memory issues**: Check for memory leaks with valgrind 3. **Compiler optimizations**: Verify release mode compilation ### Performance Tips 1. **CPU isolation**: Use \`taskset\` for CPU pinning 2. **System load**: Run benchmarks on idle system 3. **Memory frequency**: Use consistent memory settings 4. **Compiler flags**: Use \`-O3 -march=native\` for maximum performance ## Resources - [Google Benchmark Documentation](https://github.com/google/benchmark) - [Performance Optimization Guide](https://github.com/google/benchmark/blob/main/docs/user_guide.md) - [Benchmark Best Practices](https://github.com/google/benchmark/blob/main/docs/perf_counters.md)`; } static generateBenchmarkCI(projectName) { return `name: Performance Benchmarks on: push: branches: [ main, develop ] pull_request: branches: [ main ] schedule: # Run nightly benchmarks at 2 AM UTC - cron: '0 2 * * *' env: BUILD_TYPE: Release jobs: benchmark: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch full history for comparison - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y cmake ninja-build gcc-11 g++-11 python3-pip pip3 install matplotlib pandas scipy - name: Configure CMake run: | cmake -B build \\ -DCMAKE_BUILD_TYPE=\\${{ env, : .BUILD_TYPE }} \\ -DCMAKE_CXX_COMPILER=g++-11 \\ -DCMAKE_C_COMPILER=gcc-11 \\ -GNinja - name: Build benchmarks run: cmake --build build --target ${projectName}_benchmark - name: Download baseline results if: github.event_name == 'pull_request' continue-on-error: true run: | # Download baseline from main branch artifact gh api repos/\\${{ github, : .repository }}/actions/artifacts \\ --jq '.artifacts[] | select(.name=="benchmark-baseline") | .archive_download_url' \\ | head -1 \\ | xargs curl -L -H "Authorization: token \\${{ secrets, : .GITHUB_TOKEN }}" \\ -o baseline.zip unzip -q baseline.zip || echo "No baseline found" env: GITHUB_TOKEN: \\${{ secrets, : .GITHUB_TOKEN }} - name: Run benchmarks run: | # Set CPU performance mode echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor || true # Run benchmarks mkdir -p benchmark_results ./build/benchmarks/${projectName}_benchmark \\ --benchmark_format=json \\ --benchmark_out=benchmark_results/current.json \\ --benchmark_repetitions=3 \\ --benchmark_display_aggregates_only=true \\ --benchmark_counters_tabular=true - name: Analyze results if: github.event_name == 'pull_request' run: | if [ -f baseline.json ]; then python3 scripts/benchmark_analysis.py \\ baseline.json \\ benchmark_results/current.json \\ --output benchmark_results/comparison.md # Add results to PR comment echo "## Benchmark Results" >> \\$GITHUB_STEP_SUMMARY cat benchmark_results/comparison.md >> \\$GITHUB_STEP_SUMMARY else echo "No baseline found, skipping comparison" >> \\$GITHUB_STEP_SUMMARY python3 scripts/benchmark_analysis.py \\ benchmark_results/current.json \\ --output benchmark_results/analysis.md cat benchmark_results/analysis.md >> \\$GITHUB_STEP_SUMMARY fi - name: Check for regressions if: github.event_name == 'pull_request' run: | if [ -f benchmark_results/comparison.md ]; then # Check for performance regressions if grep -q "REGRESSION.*🔴" benchmark_results/comparison.md; then echo "::warning::Performance regressions detected!" grep "REGRESSION.*🔴" benchmark_results/comparison.md || true fi # Fail on major regressions (>25%) if grep -q "Change: +[2-9][0-9]\\." benchmark_results/comparison.md; then echo "::error::Major performance regression detected (>20%)" exit 1 fi fi - name: Upload benchmark results uses: actions/upload-artifact@v4 with: name: benchmark-results-\\${{ github, : .sha }} path: benchmark_results/ retention-days: 30 - name: Save baseline for main branch if: github.ref == 'refs/heads/main' uses: actions/upload-artifact@v4 with: name: benchmark-baseline path: benchmark_results/current.json retention-days: 90 - name: Generate performance report if: github.event_name == 'schedule' run: | # Generate comprehensive nightly report python3 scripts/benchmark_analysis.py \\ benchmark_results/current.json \\ --output benchmark_results/nightly_report.md # Add historical comparison if available if [ -f benchmark_results/history.json ]; then python3 scripts/benchmark_analysis.py \\ benchmark_results/history.json \\ benchmark_results/current.json \\ --output benchmark_results/historical_comparison.md fi - name: Notify on performance issues if: failure() uses: actions/github-script@v7 with: script: | github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: '⚠️ **Performance benchmark failed!** Please check the benchmark results and address any regressions.' }) benchmark-comparison: runs-on: ubuntu-latest if: github.event_name == 'pull_request' needs: benchmark steps: - name: Download current results uses: actions/download-artifact@v4 with: name: benchmark-results-\\${{ github, : .sha }} path: current_results/ - name: Download baseline results uses: actions/download-artifact@v4 with: name: benchmark-baseline path: baseline_results/ continue-on-error: true - name: Compare and comment if: success() uses: actions/github-script@v7 with: script: | const fs = require('fs'); try { const comparison = fs.readFileSync('current_results/comparison.md', 'utf8'); await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: comparison }); } catch (error) { console.log('No comparison results found'); }`; } static generateBenchmarkCMakeModule() { return `# Google Benchmark Integration Module # Provides functions for easy benchmark integration function(add_benchmark_executable TARGET_NAME) cmake_parse_arguments( BENCH "" "OUTPUT_NAME" "SOURCES;LIBRARIES;INCLUDES" \${ARGN} ) if(NOT BENCH_OUTPUT_NAME) set(BENCH_OUTPUT_NAME "\${TARGET_NAME}") endif() # Create benchmark executable add_executable(\${TARGET_NAME} \${BENCH_SOURCES}) # Set output name set_target_properties(\${TARGET_NAME} PROPERTIES OUTPUT_NAME "\${BENCH_OUTPUT_NAME}" RUNTIME_OUTPUT_DIRECTORY "\${CMAKE_BINARY_DIR}/benchmarks" ) # Link libraries target_link_libraries(\${TARGET_NAME} PRIVATE benchmark::benchmark) if(BENCH_LIBRARIES) target_link_libraries(\${TARGET_NAME} PRIVATE \${BENCH_LIBRARIES}) endif() # Include directories if(BENCH_INCLUDES) target_include_directories(\${TARGET_NAME} PRIVATE \${BENCH_INCLUDES}) endif() # Optimization flags target_compile_options(\${TARGET_NAME} PRIVATE \$<\$<CXX_COMPILER_ID:GNU>:-O3 -march=native -mtune=native -DNDEBUG> \$<\$<CXX_COMPILER_ID:Clang>:-O3 -march=native -mtune=native -DNDEBUG> \$<\$<CXX_COMPILER_ID:MSVC>:/O2 /DNDEBUG> ) # Benchmark-specific compile definitions target_compile_definitions(\${TARGET_NAME} PRIVATE BENCHMARK_ENABLE_MEMORY_COUNTERS=1 BENCHMARK_ENABLE_THREADING=1 ) # Add to benchmark target group set_target_properties(\${TARGET_NAME} PROPERTIES FOLDER "Benchmarks") endfunction() function(add_benchmark_test BENCHMARK_TARGET) cmake_parse_arguments( TEST "" "NAME;FILTER;REPETITIONS" "ARGS" \${ARGN} ) if(NOT TEST_NAME) set(TEST_NAME "benchmark_\${BENCHMARK_TARGET}") endif() if(NOT TEST_REPETITIONS) set(TEST_REPETITIONS 3) endif() set(BENCHMARK_ARGS "") if(TEST_FILTER) list(APPEND BENCHMARK_ARGS "--benchmark_filter=\${TEST_FILTER}") endif() if(TEST_REPETITIONS GREATER 1) list(APPEND BENCHMARK_ARGS "--benchmark_repetitions=\${TEST_REPETITIONS}") list(APPEND BENCHMARK_ARGS "--benchmark_display_aggregates_only=true") endif() if(TEST_ARGS) list(APPEND BENCHMARK_ARGS \${TEST_ARGS}) endif() add_test( NAME \${TEST_NAME} COMMAND \${BENCHMARK_TARGET} \${BENCHMARK_ARGS} WORKING_DIRECTORY \${CMAKE_BINARY_DIR} ) # Set timeout for benchmarks (default 5 minutes) set_tests_properties(\${TEST_NAME} PROPERTIES TIMEOUT 300) endfunction() # Helper function to discover and register all benchmarks function(discover_benchmarks) if(TARGET benchmark::benchmark) file(GLOB_RECURSE BENCHMARK_SOURCES "\${CMAKE_CURRENT_SOURCE_DIR}/*benchmark*.cpp") foreach(BENCHMARK_SOURCE \${BENCHMARK_SOURCES}) get_filename_component(BENCHMARK_NAME \${BENCHMARK_SOURCE} NAME_WE) # Skip if target already exists if(NOT TARGET \${BENCHMARK_NAME}) add_benchmark_executable(\${BENCHMARK_NAME} SOURCES \${BENCHMARK_SOURCE} LIBRARIES \${PROJECT_NAME}_lib ) add_benchmark_test(\${BENCHMARK_NAME}) endif() endforeach() endif() endfunction() # Configuration helper function(configure_benchmark_environment) # Create benchmark output directory file(MAKE_DIRECTORY "\${CMAKE_BINARY_DIR}/benchmarks") file(MAKE_DIRECTORY "\${CMAKE_BINARY_DIR}/benchmark_results") # Copy benchmark scripts if(EXISTS "\${CMAKE_CURRENT_SOURCE_DIR}/scripts/run_benchmarks.sh") configure_file( "\${CMAKE_CURRENT_SOURCE_DIR}/scripts/run_benchmarks.sh" "\${CMAKE_BINARY_DIR}/run_benchmarks.sh" COPYONLY ) # Make executable on Unix systems if(UNIX) file(CHMOD "\${CMAKE_BINARY_DIR}/run_benchmarks.sh" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) endif() endif() # Copy analysis scripts if(EXISTS "\${CMAKE_CURRENT_SOURCE_DIR}/scripts/benchmark_analysis.py") configure_file( "\${CMAKE_CURRENT_SOURCE_DIR}/scripts/benchmark_analysis.py" "\${CMAKE_BINARY_DIR}/benchmark_analysis.py" COPYONLY ) endif() endfunction() # Performance testing integration function(add_performance_test TARGET_NAME BASELINE_FILE) cmake_parse_arguments( PERF "" "THRESHOLD;OUTPUT_FILE" "BENCHMARKS" \${ARGN} ) if(NOT PERF_THRESHOLD) set(PERF_THRESHOLD 1.10) # 10% regression threshold endif() if(NOT PERF_OUTPUT_FILE) set(PERF_OUTPUT_FILE "performance_test_results.json") endif() # Create performance test add_test( NAME \${TARGET_NAME} COMMAND python3 benchmark_analysis.py \${BASELINE_FILE} \${PERF_OUTPUT_FILE} --threshold \${PERF_THRESHOLD} WORKING_DIRECTORY \${CMAKE_BINARY_DIR} ) # Set as performance test category set_tests_properties(\${TARGET_NAME} PROPERTIES LABELS "Performance" TIMEOUT 600 ) endfunction()`; } static generateBenchmarkUtils() { return `/** * Benchmark Utilities Header * Common functions and helpers for benchmark implementations */ #pragma once #include <benchmark/benchmark.h> #include <vector> #include <string> #include <memory> #include <random> #include <chrono> namespace BenchmarkUtils { // Random data generation std::vector<int> GenerateRandomInts(size_t count, int min_val = 0, int max_val = 100000); std::vector<std::string> GenerateRandomStrings(size_t count, size_t avg_length = 20); std::string GenerateRandomString(size_t length); // String utilities std::string JoinStrings(const std::vector<std::string>& strings, const std::string& delimiter); void ProcessJsonData(const std::string& json); // Mock API utilities struct MockConnection { int id; bool is_connected; std::chrono::system_clock::time_point created_at; }; struct MockRequest { std::string method; std::string path; std::string body; std::map<std::string, std::string> headers; }; struct MockResponse { int status_code; std::string body; std::map<std::string, std::string> headers; std::chrono::microseconds processing_time; }; std::shared_ptr<MockConnection> CreateMockConnection(); std::vector<std::string> ExecuteMockQuery( std::shared_ptr<MockConnection> conn, const std::string& query, const std::vector<std::string>& params ); MockRequest CreateMockRequest( const std::string& method, const std::string& path, const std::string& body = "" ); MockResponse ProcessRequest(const MockRequest& request); MockResponse ProcessRequestConcurrent(const MockRequest& request); // Server simulation void InitializeServer(); void ShutdownServer(); // Authentication utilities struct AuthService { std::string secret_key; std::chrono::seconds token_lifetime; }; std::shared_ptr<AuthService> CreateAuthService(); std::string GenerateJWT(); bool ValidateToken(std::shared_ptr<AuthService> service, const std::string& token); // Test data generation struct TestObject { int id; std::string name; std::vector<double> values; std::map<std::string, std::string> metadata; }; std::vector<TestObject> GenerateTestObjects(size_t count); std::string SerializeToJson(const std::vector<TestObject>& objects); std::vector<TestObject> DeserializeFromJson(const std::string& json); std::string GenerateTestJsonArray(size_t count); std::string GenerateTestPayload(size_t size_bytes); // Memory utilities template<typename T> class AlignedVector { public: explicit AlignedVector(size_t size, size_t alignment = 64) : size_(size), alignment_(alignment) { data_ = static_cast<T*>(std::aligned_alloc(alignment_, size_ * sizeof(T))); if (!data_) { throw std::bad_alloc(); } } ~AlignedVector() { std::free(data_); } T* data() { return data_; } const T* data() const { return data_; } size_t size() const { return size_; } T& operator[](size_t index) { return data_[index]; } const T& operator[](size_t index) const { return data_[index]; } private: T* data_; size_t size_; size_t alignment_; }; // Cache simulation utilities void FlushCaches(); void WarmupCaches(void* data, size_t size); // CPU utilities void PinToCore(int core_id); void SetHighPriority(); void DisableTurboBoost(); // Timing utilities class HighResolutionTimer { public: void Start(); void Stop(); std::chrono::nanoseconds Elapsed() const; void Reset(); private: std::chrono::high_resolution_clock::time_point start_time_; std::chrono::high_resolution_clock::time_point end_time_; bool is_running_ = false; }; // Statistics utilities struct BenchmarkStats { double mean; double median; double std_dev; double min_val; double max_val; size_t sample_count; }; BenchmarkStats CalculateStats(const std::vector<double>& values); void PrintStats(const BenchmarkStats& stats, const std::string& name); // Hardware detection struct SystemInfo { std::string cpu_model; size_t cpu_cores; size_t cpu_threads; size_t cache_l1_size; size_t cache_l2_size; size_t cache_l3_size; size_t memory_size; bool has_avx; bool has_avx2; bool has_sse4_2; }; SystemInfo GetSystemInfo(); void PrintSystemInfo(const SystemInfo& info); } // namespace BenchmarkUtils`; } static generateBenchmarkUtilsImpl() { return `/** * Benchmark Utilities Implementation * Common functions and helpers for benchmark implementations */ #include "benchmark_utils.hpp" #include <algorithm> #include <numeric> #include <random> #include <sstream> #include <thread> #include <mutex> #include <condition_variable> #include <atomic> #include <cstring> #include <iostream> #ifdef __linux__ #include <sched.h> #include <sys/resource.h> #include <unistd.h> #endif namespace BenchmarkUtils { // Global random number generator thread_local std::mt19937 g_rng(std::random_device{}()); std::vector<int> GenerateRandomInts(size_t count, int min_val, int max_val) { std::vector<int> result; result.reserve(count); std::uniform_int_distribution<int> dist(min_val, max_val); for (size_t i = 0; i < count; ++i) { result.push_back(dist(g_rng)); } return result; } std::string GenerateRandomString(size_t length) { const std::string chars = "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ