loqatevars/lib/utils.js

Locate JavaScript files with 'const' or 'process.env' usage in LLM-generated codebases

/**
 * @file This file contains the core utilities for the `loqatevars` tool.
 * It includes functions for scanning directories, analyzing JavaScript files for specific
 * code patterns (`const` variables and `process.env` usage), and managing concurrent
 * operations. The utilities are designed to be efficient and scalable, using techniques
 * like AST parsing and asynchronous processing.
 *
 * The main functions exported are:
 * - `findMatchingFiles`: Performs a basic scan and returns a list of matching file paths.
 * - `findMatchingFilesDetailed`: Provides a more comprehensive analysis with detailed statistics.
 * - `analyzeConstUsage`: The core analysis function that uses `acorn` to parse JavaScript
 *   and identify relevant code patterns.
 *
 * Scalability Considerations:
 * - The `asyncPool` function is used to limit concurrency and prevent the application
 *   from overwhelming the system with too many simultaneous file operations.
 * - The use of `globby` streams for file searching is a memory-efficient approach,
 *   especially for large directories.
 * - The AST parsing in `analyzeConstUsage` can be CPU-intensive, so the concurrency
 *   limit is based on the number of CPU cores to optimize performance.
 */

const fs = require('fs-extra');
const path = require('path');
const localVars = require('../config/localVars');
const os = require('os');
const acorn = require('acorn');
const walk = require('acorn-walk');
const { AppError } = require('./errors');
const { utils: d } = require('./logger');

/**
 * `asyncPool` is a utility function for running a collection of asynchronous tasks
 * with a specified concurrency limit. This is crucial for managing system resources
 * and preventing performance degradation when dealing with a large number of I/O-bound
 * or CPU-bound operations.
 *
 * The function works by maintaining a pool of executing promises and using `Promise.race`
 * to wait for the next available slot in the pool. This ensures that no more than `limit`
 * tasks are running at any given time.
 *
 * @param {number} limit - The maximum number of tasks to run concurrently.
 * @param {Array<Function>} tasks - An array of functions that return promises.
 * @returns {Promise<Array<object>>} A promise that resolves to an array of settlement objects,
 *                                    similar to `Promise.allSettled`.
 */
async function asyncPool(limit, tasks) {
  if (!Number.isInteger(limit) || limit <= 0) { // reject invalid limit early
    throw new AppError('Invalid concurrency limit', 'INVALID_POOL_LIMIT');
  }
  const executing = [];
  const results = [];

  for (const task of tasks) {
    const p = Promise.resolve().then(task); // ensure task is a promise
    results.push(p);
    const e = p.finally(() => { // cleanup regardless of outcome
      const idx = executing.indexOf(e); // ensure promise is in the list
      if (idx > -1) executing.splice(idx, 1); // remove only if present
    });
    executing.push(e);
    if (executing.length >= limit) {
      await Promise.race(executing); // wait for the fastest task to finish
    }
  }

  return Promise.allSettled(results);
}

/**
 * The concurrency level is determined based on the number of available CPU cores.
 * This is a common strategy for optimizing the performance of parallel tasks,
 * as it allows the application to take full advantage of the available processing power.
 *
 * In environments where the number of CPU cores cannot be determined, a default
 * concurrency of 1 is used to ensure stability.
 */
const cpuInfo = os.cpus();
const concurrency = cpuInfo ? Math.max(cpuInfo.length, 1) : 1;


/**
 * Analyzes the provided JavaScript code content to identify usage of `const` declarations
 * and `process.env`. It uses AST (Abstract Syntax Tree) parsing for accuracy.
 *
 * The function distinguishes between `const` declarations used for module imports (`require`)
 * and those used for defining variables. This is important because we only want to flag
 * variable declarations.
 *
 * @param {string} content - The JavaScript code to analyze.
 * @returns {object} An object containing the analysis results, including counts of
 *                   different `const` types and a flag indicating if `process.env` is used.
 */
/**
 * `createScopeTracker` is a factory function that creates a scope tracking utility.
 * This utility is essential for accurately analyzing variable declarations and references
 * within the AST. It uses a stack of maps to manage lexical scopes, allowing for
 * correct resolution of variable bindings.
 *
 * The tracker provides methods for entering and exiting scopes, adding new bindings,
 * and finding existing bindings. This is particularly important for distinguishing
 * between top-level `const` declarations and those within nested scopes.
 *
 * @returns {object} A scope tracker object with methods for scope management.
 */
function createScopeTracker() {
  const scopeStack = [new Map()];

  return {
    enterScope() {
      scopeStack.push(new Map());
    },
    exitScope() {
      scopeStack.pop();
    },
    addBinding(name, value) {
      scopeStack[scopeStack.length - 1].set(name, value);
    },
    findBinding(name) {
      for (let i = scopeStack.length - 1; i >= 0; i--) {
        if (scopeStack[i].has(name)) {
          return scopeStack[i].get(name);
        }
      }
      return undefined;
    },
    isTopLevel() {
      return scopeStack.length === 1;
    }
  };
}

// Helper to identify `env` property access
function isEnvProperty(prop) {
  return (
    (prop.type === 'Identifier' && prop.name === 'env') ||
    (prop.type === 'Literal' && prop.value === 'env')
  );
}

/**
 * `analyzeConstUsage` is the core analysis function of the `loqatevars` tool.
 * It takes JavaScript code as input and uses AST (Abstract Syntax Tree) parsing
 * to identify `const` declarations and `process.env` usage.
 *
 * The function is designed to be highly accurate, distinguishing between `const`
 * declarations for module imports and those for regular variables. It also handles
 * various forms of `process.env` access, including direct and indirect references.
 *
 * @param {string} content - The JavaScript code to analyze.
 * @param {string} filePath - The path to the file being analyzed (used for logging).
 * @returns {object} An object containing the analysis results.
 */
function analyzeConstUsage(content, filePath) {
  let totalConst = 0;
  let importConst = 0;
  let variableConst = 0;
  let hasProcessEnv = false;

  try {
    let ast;
    try {
      // First, try parsing the code as an ES module. This is the modern standard
      // and should be the default for most new JavaScript code.
      ast = acorn.parse(content, {
        ecmaVersion: 'latest',
        sourceType: 'module',
        locations: true,
      });
    } catch (parseError) {
      try {
        // If module parsing fails, fall back to parsing as a script. This is
        // necessary for older codebases or files that are not ES modules.
        ast = acorn.parse(content, {
          ecmaVersion: 'latest',
          sourceType: 'script',
          locations: true,
        });
      } catch (scriptParseError) {
        // If both parsing attempts fail, throw a detailed error to help with debugging.
        throw new AppError(
          `Failed to parse ${filePath}: Both module and script parsing failed. Module error: ${parseError.message}, Script error: ${scriptParseError.message}`,
          'AST_PARSING_ERROR'
        );
      }
    }

    const scope = createScopeTracker();

    // `walk.recursive` traverses the AST, allowing us to inspect each node.
    // We provide custom visitor functions for the node types we're interested in.
    walk.recursive(ast, null, {
      BlockStatement(node, state, c) {
        scope.enterScope();
        walk.base.BlockStatement(node, state, c); // traverse inner nodes
        scope.exitScope();
      },
      Function(node, state, c) {
        scope.enterScope();
        walk.base.Function(node, state, c); // traverse function body
        scope.exitScope();
      },
      ImportDeclaration(node) {
        // `import` statements are a form of `const` declaration, but we want to
        // distinguish them from variable declarations.
        importConst += node.specifiers.length;
      },
      VariableDeclaration(node, state, c) {
        if (node.kind === 'const') {
          node.declarations.forEach(decl => {
            totalConst++;
            // We only flag `const` declarations at the top level of the module,
            // as these are more likely to be environment-specific configurations.
            if (scope.isTopLevel()) {
              if (decl.init && decl.init.type === 'CallExpression' && decl.init.callee.name === 'require') {
                importConst++;
              } else {
                variableConst++;
              }
            }
            // Track variables that are assigned the `process` object, so we can
            // detect indirect `process.env` access later.
            if (decl.id.type === 'Identifier' && decl.init) {
              if (decl.init.type === 'Identifier' && decl.init.name === 'process') {
                scope.addBinding(decl.id.name, 'process');
              }
            }
          });
        }
        walk.base.VariableDeclaration(node, state, c);
      },
      MemberExpression(node, state, c) {
        // This visitor function detects direct and indirect access to `process.env`.
        if (
          node.object.type === 'Identifier' &&
          node.object.name === 'process' &&
          isEnvProperty(node.property)
        ) {
          hasProcessEnv = true;
          d('Found process.env at %s:%d:%d', filePath, node.loc.start.line, node.loc.start.column);
        } else if (node.object.type === 'Identifier') {
          const binding = scope.findBinding(node.object.name);
          if (isEnvProperty(node.property) && binding === 'process') {
            hasProcessEnv = true;
            d('Found indirect env access through %s at %s:%d:%d',
              node.object.name, filePath, node.loc.start.line, node.loc.start.column);
          } else if (binding === 'process.env') {
            hasProcessEnv = true;
            d('Found indirect env variable %s at %s:%d:%d',
              node.object.name, filePath, node.loc.start.line, node.loc.start.column);
          }
        }
        walk.base.MemberExpression(node, state, c);
      },
      VariableDeclarator(node, state, c) {
        // This visitor handles cases where `process` or `process.env` are
        // assigned to other variables, which is a form of indirect access.
        if (node.id.type === "Identifier" && node.init) {
          if (node.init.type === "Identifier") {
            if (node.init.name === "process") {
              scope.addBinding(node.id.name, "process");
            } else {
              const ref = scope.findBinding(node.init.name);
              if (ref === "process" || ref === "process.env") {
                scope.addBinding(node.id.name, ref);
              }
            }
          } else if (node.init.type === "MemberExpression" && node.init.object.type === "Identifier" && node.init.object.name === "process" && isEnvProperty(node.init.property)) {
            scope.addBinding(node.id.name, "process.env");
          }
        }
        // This handles destructuring assignments from `process`, such as
        // `const { env } = process;`.
        if (node.id.type === 'ObjectPattern' && node.init && node.init.type === 'Identifier' && node.init.name === 'process') {
          const hasEnv = node.id.properties.some(
            prop => (prop.key.name === 'env' || (prop.value && prop.value.name === 'env'))
          );
          if (hasEnv) {
            hasProcessEnv = true;
            d('Found process destructuring at %s:%d:%d', filePath, node.loc.start.line, node.loc.start.column);
          }
        }
        walk.base.VariableDeclarator(node, state, c);
      }
    });
  } catch (error) {
    // Re-throw AppError instances directly
    if (error instanceof AppError) {
      throw error;
    }
    // Wrap all other errors in AppError with consistent formatting
    throw new AppError(`ANALYSIS_ERROR: Failed to analyze ${filePath} - ${error.message}`, 'ANALYSIS_ERROR');
  }

  return {
    totalConst,
    importConst,
    variableConst,
    hasProcessEnv,
    shouldFlag: hasProcessEnv || variableConst > 0
  };
}

/**
 * Uses `globby` to find all files in a directory that match the given extensions,
 * while respecting a list of ignored files and directories. This function is a wrapper
 * around `globby` to simplify its usage in this application.
 *
 * @param {string} dir - The directory to search in.
 * @param {string[]} extensions - An array of file extensions to look for (e.g., ['.js']).
 * @param {string|string[]} [ignoreFiles=[]] - A single file or an array of files/patterns to ignore.
 * @returns {string[]} An array of absolute file paths that match the criteria.
 */
async function searchFiles(dir, extensions, ignoreFiles = []) {
 const ignoreList = ignoreFiles
   ? (Array.isArray(ignoreFiles) ? ignoreFiles : [ignoreFiles])
   : []; // default to empty array when ignoreFiles omitted
 
  // Validate and normalize extensions
  extensions = extensions.map(ext => {
    if (typeof ext !== 'string') { // ensure each extension is a string before further checks
      throw new AppError('INVALID_EXTENSION_TYPE', 'INVALID_EXTENSION_TYPE');
    }
    if (ext === '') { // reject empty strings early
      throw new AppError('Extension cannot be empty', 'INVALID_EXTENSION');
    }
    if (!ext.startsWith('.')) {
      ext = '.' + ext;
    }
    if (ext === '.') { // '.' alone is not a valid extension
      throw new AppError('Extension cannot be just a dot', 'INVALID_EXTENSION');
    }
    if (ext.includes('/') || ext.includes('\\')) { // no path characters allowed
      throw new AppError(`Extension cannot contain path separators: ${ext}`, 'INVALID_EXTENSION');
    }
    return ext;
  });
 
 // Use multiple patterns instead of brace expansion
 const patterns = extensions.map(ext => `**/*${ext}`);
 
 const ignorePatterns = [
   ...ignoreList,
   ...localVars.ignoreDirs.map(d => `**/${d}/**`)
 ];

  // Use dynamic import so we can load ESM globby within CommonJS
  let streamFn;
  try {
    const globbyModule = await import('globby');
    // Determine the stream function regardless of where it is exported from
    streamFn =
      globbyModule.stream ||
      (globbyModule.default && globbyModule.default.stream) ||
      globbyModule.globbyStream;
    if (typeof streamFn !== 'function') {
      throw new Error('Stream function not found');
    }
  } catch (err) {
    throw new AppError(`GLOBBY_IMPORT_ERROR: ${err.message}`, 'GLOBBY_IMPORT_ERROR');
  }
 const fileStream = streamFn(patterns, {
    cwd: dir,
    ignore: ignorePatterns,
    onlyFiles: true,
    absolute: true,
  });

 const files = [];
 for await (const entry of fileStream) { files.push(entry); }

 return files;
}

/**
 * Asynchronously validates that a given path is a valid, existing directory.
 * This is a crucial step before attempting to scan a directory, as it prevents
 * errors from occurring later in the process.
 *
 * @param {string} dir - The directory path to validate.
 * @throws {AppError} If the directory is invalid, not found, or not a directory.
 */
async function validateDirectory(dir) {
  if (typeof dir !== 'string') {
    throw new AppError(`INVALID_DIRECTORY: The directory '${dir}' is not a valid string.`, 'INVALID_DIRECTORY');
  }

  d('Validating directory: %s', dir);

  try {
    const stat = await fs.stat(dir);
    d('Is directory: %s', stat.isDirectory());
    if (!stat.isDirectory()) {
      throw new AppError(`PATH_NOT_DIRECTORY: The path '${dir}' is not a directory.`, 'PATH_NOT_DIRECTORY');
    }
  } catch (error) {
    if (error.code === 'ENOENT') {
      throw new AppError(`DIRECTORY_NOT_FOUND: The directory '${dir}' does not exist.`, 'DIRECTORY_NOT_FOUND');
    }
    // Re-throw AppError instances directly
    if (error instanceof AppError) {
      throw error;
    }
    // Wrap all other errors in AppError with consistent formatting
    throw new AppError(`VALIDATION_ERROR: Failed to validate directory '${dir}' - ${error.message}`, 'VALIDATION_ERROR');
  }
}

/**
 * Scans a directory for JavaScript files that contain either `const` variable declarations
 * (excluding `require` imports) or `process.env` usage. This is the main "simple" scan
 * function of the tool.
 *
 * It orchestrates the process of validating the directory, searching for files,
 * and analyzing them. The results are returned as an array of file paths.
 *
 * @param {string} dir - The directory to scan.
 * @param {string|string[]} ignoreFiles - Files/patterns to ignore.
 * @param {string[]} extensions - File extensions to scan.
 * @returns {Promise<string[]>} A promise that resolves to an array of matching file paths.
 */
// '**/localVars.js' ensures the default ignore pattern matches any directory level
async function findMatchingFiles(dir = process.cwd(), ignoreFiles = '**/localVars.js', extensions = ['.js']) {
  await validateDirectory(dir);

  const files = await searchFiles(dir, extensions, ignoreFiles);
  const matches = [];
  const tasks = files.map(fullPath => async () => {
    try {
      const content = await fs.readFile(fullPath, 'utf8');
      const analysis = analyzeConstUsage(content, fullPath);
      if (analysis.shouldFlag) {
        return path.relative(dir, fullPath);
      }
    } catch (error) {
      d('Skipping file due to error: %s', fullPath, error);
      if (!(error instanceof AppError)) {
        console.error(`FILE_PROCESSING_ERROR: Failed to process ${fullPath} - ${error.message}`);
      }
    }
    return null;
  });

  const results = await asyncPool(concurrency, tasks);

  results.forEach(result => {
    if (result.status === 'fulfilled' && result.value) {
      matches.push(result.value);
    } else if (result.status === 'rejected') {
      // This part should ideally not be reached if errors are caught inside the limit wrapper
      d('An unexpected error occurred: %s', result.reason);
    }
  });

  return matches;
}

/**
 * Scans a directory and returns a detailed report of files that contain `const`
 * variable declarations or `process.env` usage. This is the "detailed" scan function.
 *
 * Similar to `findMatchingFiles`, but it returns a much richer set of data, including
 * statistics about the scan and detailed information about each matching file.
 *
 * @param {string} dir - The directory to scan.
 * @param {string|string[]} ignoreFiles - Files/patterns to ignore.
 * @param {string[]} extensions - File extensions to scan.
 * @returns {Promise<object>} A promise that resolves to an object containing the detailed
 *                            scan results.
 */
// '**/localVars.js' ensures all localVars.js files are skipped regardless of directory depth
async function findMatchingFilesDetailed(dir = process.cwd(), ignoreFiles = '**/localVars.js', extensions = ['.js']) {
  await validateDirectory(dir);

  const files = await searchFiles(dir, extensions, ignoreFiles);
  const matches = [];
  const tasks = files.map(fullPath => async () => {
    try {
      const content = await fs.readFile(fullPath, 'utf8');
      const analysis = analyzeConstUsage(content, fullPath);
      d('Scanned file: %s, Analysis result: %o', fullPath, analysis);

      if (analysis.shouldFlag) {
        const relativePath = path.relative(dir, fullPath);
        return {
          relativePath: relativePath,
          hasProcessEnv: analysis.hasProcessEnv,
          totalConst: analysis.totalConst,
          importConst: analysis.importConst,
          variableConst: analysis.variableConst,
          reason: analysis.hasProcessEnv && analysis.variableConst > 0 ? 'both' :
                 analysis.hasProcessEnv ? 'process.env' : 'variables'
        };
      }
    } catch (error) {
      d('Skipping file due to error: %s', fullPath, error);
      if (!(error instanceof AppError)) {
        console.error(`FILE_PROCESSING_ERROR: Failed to process ${fullPath} - ${error.message}`);
      }
    }
    return null;
  });

  const results = await asyncPool(concurrency, tasks);

  results.forEach(result => {
    if (result.status === 'fulfilled' && result.value) {
      matches.push(result.value);
      d('Flagged file: %s', result.value.relativePath);
    } else if (result.status === 'rejected') {
      d('An unexpected error occurred: %s', result.reason);
    }
  });

  return {
    matches,
    summary: {
      scannedFiles: files.length,
      matchingFiles: matches.length,
      ignoredFiles: Array.isArray(ignoreFiles) ? ignoreFiles : [ignoreFiles]
    }
  };
}

module.exports = {
  analyzeConstUsage,
  findMatchingFiles,
  findMatchingFilesDetailed,
  searchFiles,
  validateDirectory,
  asyncPool,
  concurrency
};