eslint-rule-benchmark/dist/runners/run-benchmarks-from-config.js

Benchmark ESLint rules with detailed performance metrics for CI and plugin development

import fs from 'node:fs/promises'
import path from 'node:path'
import {
  DEFAULT_TIMEOUT_MS,
  DEFAULT_ITERATIONS,
  DEFAULT_WARMUP_ENABLED,
  DEFAULT_WARMUP_ITERATIONS,
  DEFAULT_SEVERITY,
} from '../constants/index.js'
import { getLanguageByFileName } from '../core/utilities/get-language-by-file-name.js'
import { isSupportedExtension } from '../core/utilities/is-supported-extension.js'
import { getFileExtension } from '../core/utilities/get-file-extension.js'
import { createTestCase } from '../core/test-case/create-test-case.js'
import { runBenchmark } from '../core/benchmark/run-benchmark.js'
import { runReporters } from '../reporters/run-reporters.js'
async function runBenchmarksFromConfig(parameters) {
  let { eslintConfigFile, reporterOptions, configDirectory, userConfig } =
    parameters
  if (userConfig.tests.length === 0) {
    console.warn('User configuration contains no tests. Exiting.')
    return
  }
  let allTestSpecResults = []
  let allTestCasePreparationTasks = userConfig.tests.map(async testSpec => {
    let specBenchmarkConfig = {
      warmup: {
        iterations:
          testSpec.warmup?.iterations ??
          userConfig.warmup?.iterations ??
          DEFAULT_WARMUP_ITERATIONS,
        enabled:
          testSpec.warmup?.enabled ??
          userConfig.warmup?.enabled ??
          DEFAULT_WARMUP_ENABLED,
      },
      iterations:
        testSpec.iterations ?? userConfig.iterations ?? DEFAULT_ITERATIONS,
      timeout: testSpec.timeout ?? userConfig.timeout ?? DEFAULT_TIMEOUT_MS,
      reporters: reporterOptions,
      name: testSpec.name,
    }
    let caseProcessingPromises = testSpec.cases.map(
      async (caseItem, caseIndex) => {
        try {
          let codeSamples = await loadCodeSamples(
            caseItem.testPath,
            configDirectory,
          )
          let ruleConfig = {
            severity: caseItem.severity ?? DEFAULT_SEVERITY,
            options: caseItem.options,
            ruleId: testSpec.ruleId,
            path: testSpec.rulePath,
          }
          let caseNameSuffix = `Case ${caseIndex + 1}`
          let testCaseName = `${testSpec.name} - ${caseNameSuffix}`
          let testCaseId = `config-test-${testSpec.name.replaceAll(/\s+/gu, '-')}-case-${caseIndex}-${Date.now()}`
          return createTestCase({
            samples: codeSamples,
            name: testCaseName,
            rule: ruleConfig,
            id: testCaseId,
          })
        } catch (error) {
          let errorValue = error
          console.warn(
            `Skipping case ${caseIndex + 1} in test "${testSpec.name}" due to an error: ${errorValue.message}`,
          )
          return null
        }
      },
    )
    let resolvedTestCases = await Promise.all(caseProcessingPromises)
    let validTestCases = resolvedTestCases.filter(tc => tc !== null)
    return { testCases: validTestCases, specBenchmarkConfig, testSpec }
  })
  let preparedDataForAllSpecs = await Promise.all(allTestCasePreparationTasks)
  for (let preparedData of preparedDataForAllSpecs) {
    let { specBenchmarkConfig, testCases, testSpec } = preparedData
    if (testCases.length > 0) {
      console.info(
        `Starting benchmark run for test spec "${testSpec.name}" with ${testCases.length} test case(s)...`,
      )
      let specRunSampleResults =
        // eslint-disable-next-line no-await-in-loop
        await runBenchmark({
          config: specBenchmarkConfig,
          eslintConfigFile,
          configDirectory,
          testCases,
        })
      if (specRunSampleResults && specRunSampleResults.length > 0) {
        let currentTestCaseResults = []
        for (let tc of testCases) {
          let samplesForThisTestCase = specRunSampleResults.filter(taskResult =>
            taskResult.name.startsWith(`${tc.name} on `),
          )
          if (samplesForThisTestCase.length > 0) {
            currentTestCaseResults.push({
              samplesResults: samplesForThisTestCase,
              description: tc.description,
              name: tc.name,
              rule: tc.rule,
              id: tc.id,
            })
          }
        }
        if (currentTestCaseResults.length > 0) {
          allTestSpecResults.push({
            benchmarkConfig: {
              iterations: specBenchmarkConfig.iterations,
              timeout: specBenchmarkConfig.timeout,
              warmup: specBenchmarkConfig.warmup,
            },
            testCaseResults: currentTestCaseResults,
            rulePath: testSpec.rulePath,
            ruleId: testSpec.ruleId,
            name: testSpec.name,
          })
        }
      }
    }
  }
  if (allTestSpecResults.every(spec => spec.testCaseResults.length === 0)) {
    console.error(
      'No valid test cases or benchmark results could be generated from the user configuration. Exiting.',
    )
    process.exitCode = 1
    return
  }
  if (allTestSpecResults.length > 0) {
    console.info(
      `Benchmark run completed. ${allTestSpecResults.length} test specifications processed.`,
    )
    await runReporters(allTestSpecResults, userConfig, reporterOptions)
  }
  console.info('Benchmark run finished.')
}
async function loadCodeSamples(testPath, configDirectory) {
  let pathsToProcess = Array.isArray(testPath) ? testPath : [testPath]
  let fileArrays = await Promise.all(
    pathsToProcess.map(async currentPath => {
      let filesForCurrentPath = []
      try {
        let resolvedPath = path.resolve(configDirectory, currentPath)
        let stats = await fs.stat(resolvedPath)
        if (stats.isDirectory()) {
          let filesInDirectory = await fs.readdir(resolvedPath)
          for (let fileName of filesInDirectory.filter(item =>
            isSupportedExtension(getFileExtension(item)),
          )) {
            filesForCurrentPath.push(path.join(resolvedPath, fileName))
          }
        } else if (
          stats.isFile() &&
          isSupportedExtension(getFileExtension(resolvedPath))
        ) {
          filesForCurrentPath.push(resolvedPath)
        }
      } catch (error) {
        if (error instanceof Error) {
          console.warn(
            `Warning: Could not process path ${currentPath}: ${error.message}. Skipping.`,
          )
        } else {
          console.warn(
            `Warning: Could not process path ${currentPath}: ${String(error)}. Skipping.`,
          )
        }
      }
      return filesForCurrentPath
    }),
  )
  let sourceFiles = fileArrays.flat()
  if (sourceFiles.length === 0) {
    throw new Error(
      `No supported source files found for testPath: ${JSON.stringify(testPath)}`,
    )
  }
  let codeSamples = []
  await Promise.all(
    sourceFiles.map(async file => {
      try {
        let code = await fs.readFile(file, 'utf8')
        codeSamples.push({
          language: getLanguageByFileName(file),
          filename: path.basename(file),
          code,
        })
      } catch (error) {
        if (error instanceof Error) {
          console.warn(
            `Warning: Skipping file ${file} due to read error: ${error.message}`,
          )
        } else {
          console.warn(
            `Warning: Skipping file ${file} due to read error: ${String(error)}`,
          )
        }
      }
    }),
  )
  if (codeSamples.length === 0) {
    throw new Error(
      `No valid code samples could be loaded from testPath: ${JSON.stringify(testPath)}`,
    )
  }
  return codeSamples
}
export { runBenchmarksFromConfig }