eslint-rule-benchmark/dist/core/benchmark/run-benchmark.js

Benchmark ESLint rules with detailed performance metrics for CI and plugin development

import { createESLintInstance } from '../eslint/create-eslint-instance.js'
import { calculateStatistics } from './calculate-statistics.js'
import { filterOutliers } from './filter-outliers.js'
import { createBench } from './create-bench.js'
async function runBenchmark(parameters) {
  let { eslintConfigFile, configDirectory, testCases, config } = parameters
  if (testCases.length === 0) {
    return null
  }
  let bench = createBench({
    warmupIterations: config.warmup.iterations,
    iterations: config.iterations,
    warmup: config.warmup.enabled,
    timeoutMs: config.timeout,
  })
  for (let testCase of testCases) {
    let currentTestCaseLanguages = []
    for (let sample of testCase.samples) {
      if (!currentTestCaseLanguages.includes(sample.language)) {
        currentTestCaseLanguages.push(sample.language)
      }
    }
    if (currentTestCaseLanguages.length === 0) {
      console.warn(
        `Skipping TestCase "${testCase.name}" as it has no samples with recognized languages.`,
      )
      continue
    }
    let eslint
    try {
      eslint = await createESLintInstance({
        languages: currentTestCaseLanguages,
        rule: testCase.rule,
        eslintConfigFile,
        configDirectory,
      })
      await eslint.lintText('/* eslint-disable */')
    } catch (error) {
      let errorValue = error
      console.error(
        `Failed to create ESLint instance for TestCase "${testCase.name}": ${errorValue.message}. Skipping this test case.`,
      )
      continue
    }
    for (let sample of testCase.samples) {
      bench.add(`${testCase.name} on ${sample.filename}`, async () => {
        await eslint.lintText(sample.code, { filePath: sample.filename })
      })
    }
  }
  if (bench.tasks.length === 0) {
    console.warn('No benchmark tasks were added. Nothing to run.')
    return null
  }
  let tinybenchTasks = await bench.run()
  let processedResults = []
  for (let task of tinybenchTasks) {
    if (!task.result?.samples) {
      console.warn(
        `Task "${task.name}" ran but has no samples or result. Skipping.`,
      )
      continue
    }
    let originalSamples = task.result.samples
    let { outliersRemovedCount: _outliersRemovedCount, filteredSamples } =
      filterOutliers(originalSamples)
    let samplesToProcess =
      filteredSamples.length > 0 ? filteredSamples : originalSamples
    if (filteredSamples.length === 0 && originalSamples.length > 0) {
      console.warn(
        `All ${originalSamples.length} samples for task "${task.name}" were filtered out as outliers. Statistics will be based on original samples or appear as zero if original was also empty.`,
      )
    }
    let calculatedMetrics = calculateStatistics(samplesToProcess)
    processedResults.push({
      metrics: calculatedMetrics,
      name: task.name,
    })
  }
  return processedResults.length > 0 ? processedResults : null
}
export { runBenchmark }