@codeque/core/dist/getFilesList.js

Multiline code search for every language. Structural code search for JavaScript, TypeScript, HTML and CSS

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.typeScriptFamilyExtensionTester = exports.pythonExtensionTester = exports.pathToPosix = exports.pathSeparatorChar = exports.luaExtensionTester = exports.htmlFamilyExtensionTester = exports.getFsRoot = exports.getFilesList = exports.filterIncludeExclude = exports.filterExtensions = exports.extensionTester = exports.cssExtensionTester = void 0;

var _path = _interopRequireDefault(require("path"));

var _fs = require("fs");

var _ignore = _interopRequireDefault(require("ignore"));

var _utils = require("./utils");

var _minimatch = _interopRequireDefault(require("minimatch"));

var _index = require("dpdm/lib/index.js");

var _child_process = require("child_process");

var _globEscape = _interopRequireDefault(require("glob-escape"));

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

//@ts-ignore
const typeScriptFamilyExtensionTester = /\.(js|jsx|ts|tsx|json|mjs|cjs)$/;
exports.typeScriptFamilyExtensionTester = typeScriptFamilyExtensionTester;
const htmlFamilyExtensionTester = /\.(html|htm)$/;
exports.htmlFamilyExtensionTester = htmlFamilyExtensionTester;
const cssExtensionTester = /\.(css)$/;
exports.cssExtensionTester = cssExtensionTester;
const pythonExtensionTester = /\.(py)$/;
exports.pythonExtensionTester = pythonExtensionTester;
const luaExtensionTester = /\.(lua)$/;
/**
 * @deprecated use `typeScriptFamilyExtensionTester` instead
 */

exports.luaExtensionTester = luaExtensionTester;
const extensionTester = typeScriptFamilyExtensionTester;
exports.extensionTester = extensionTester;

const pathToPosix = fsPath => fsPath.replace(/\\/g, '/');

exports.pathToPosix = pathToPosix;
const bigFileSizeInBytes = 1024 * 100; // 100 kb

const getFilesListByEntryPoint = async (root, entryPoint) => {
  // dpdm does not support custom search directory :/
  const oldProcessCwd = process.cwd;

  process.cwd = () => root;

  const tree = await (0, _index.parseDependencyTree)((0, _globEscape.default)(entryPoint), {
    context: root
  });
  process.cwd = oldProcessCwd;
  const projectFiles = Object.keys(tree).filter(file => !file.includes('node_modules') && typeScriptFamilyExtensionTester.test(file));
  return projectFiles.map(filePath => _path.default.resolve(root, filePath));
};

const directoriesBlackList = ['.git'];

const getFilesListByGitChanges = async (root, extensionTester) => {
  const {
    error,
    output
  } = (0, _child_process.spawnSync)('git', ['diff', '--name-only', 'HEAD'], {
    cwd: root
  });

  if (error) {
    throw error;
  }

  const filesList = output.filter(data => data !== null).map(buff => buff?.toString()).join('').split('\n').filter(maybeFile => maybeFile.length > 0).filter(filePath => extensionTester.test(filePath)).map(filePath => _path.default.resolve(root, filePath));
  return filesList;
};

const pathSeparatorChar = _path.default.sep;
exports.pathSeparatorChar = pathSeparatorChar;

const getFsRoot = fsPath => _path.default.parse(fsPath).root;

exports.getFsRoot = getFsRoot;

const getGitIgnoreContentForDirectory = async dirPath => {
  const fsRoot = getFsRoot(dirPath);
  const gitignorePathSeparator = '/';
  let gitignore = [];

  try {
    const gitignorePath = _path.default.join(dirPath, '.gitignore');

    const gitignoreFileContent = (await _fs.promises.readFile(gitignorePath)).toString();
    const lines = gitignoreFileContent.split('\n').map(line => line.trim());
    const nonCommentedNonEmptyLines = lines.filter(line => !/^(\s*)#/.test(line)).filter(line => !/^(\s*)$/.test(line));
    const maybeRelativeToDir = nonCommentedNonEmptyLines.map(line => {
      if ( // Pattern starts with dir separator eg. /some
      line.startsWith(gitignorePathSeparator) || // Pattern contains dir separator as not last char and is not path beginning wildcard eg. some/path, but not **/some/path
      !line.startsWith(`**${gitignorePathSeparator}`) && line.substring(0, line.length - 1).includes(gitignorePathSeparator)) {
        // pattern should be relative to .gitignore location directory
        // `ignore` does not allow absolute paths, so we have to hack by removing initial '/' or 'C:\'
        const fsPosixPathWithoutRoot = pathToPosix(dirPath.replace(fsRoot, ''));
        const composedPath = `${fsPosixPathWithoutRoot}${gitignorePathSeparator}${line}`; // We normalize cos path can end with `/` so there would be '//'
        // Also we change path to Posix, because path.normalise change it to format depending on platform

        return pathToPosix(_path.default.normalize(composedPath));
      } // pattern should not be relative to .gitignore location directory, eg. '*.json', '**/someFile', 'dist/'


      return line;
    });
    gitignore = maybeRelativeToDir;
  } catch (e) {
    e;
  }

  return gitignore;
};

const removeInitialDot = p => p.replace(/^\.(\\|\/)/, '');

const filterIncludeExclude = ({
  searchRoot,
  include: _include,
  exclude: _exclude,
  filesList
}) => {
  const include = _include ? _include.map(removeInitialDot) : _include;

  const exclude = _exclude.map(removeInitialDot);

  if (!include && exclude.length === 0) {
    return filesList;
  }

  return filesList.map(p => _path.default.relative(searchRoot, p)).filter(id => exclude.reduce((result, pattern) => result && !(0, _minimatch.default)(id, pattern), true)).filter(id => include ? include.reduce((result, pattern) => result || (0, _minimatch.default)(id, pattern), false) : true).map(p => _path.default.join(searchRoot, p));
};

exports.filterIncludeExclude = filterIncludeExclude;

const filterExtensions = (filesList, extensionTester) => {
  return filesList.filter(filePath => extensionTester.test(filePath));
};

exports.filterExtensions = filterExtensions;

const getFilesList = async ({
  searchRoot: _searchRoot,
  entryPoint = undefined,
  byGitChanges = false,
  omitGitIgnore = false,
  ignoreNodeModules = true,
  searchBigFiles = false,
  exclude = [],
  include = undefined,
  hardStopFlag,
  extensionTester = typeScriptFamilyExtensionTester
}) => {
  const searchRoot = _path.default.normalize(_searchRoot);

  const fsRoot = getFsRoot(searchRoot);
  let filesList = [];

  if (byGitChanges) {
    filesList = await getFilesListByGitChanges(searchRoot, extensionTester);
  } else if (entryPoint) {
    filesList = await getFilesListByEntryPoint(searchRoot, entryPoint);
  } else {
    const InitialIgnore = ignoreNodeModules ? ['node_modules'] : []; // Get parent to root gitignore

    if (!omitGitIgnore) {
      const searchRootSegments = searchRoot.replace(fsRoot, '').split(pathSeparatorChar);
      const pathSegmentsToSystemRoot = [];

      for (let i = 0; i < searchRootSegments.length; i++) {
        let currentPath = searchRootSegments.slice(0, i).join(pathSeparatorChar);
        currentPath = fsRoot + currentPath;
        pathSegmentsToSystemRoot.push(currentPath);
      }

      const parentDirsIgnore = (await Promise.all(pathSegmentsToSystemRoot.map(parentPath => getGitIgnoreContentForDirectory(parentPath)))).filter(parentGitignore => parentGitignore.length > 0).flat(1);
      InitialIgnore.push(...parentDirsIgnore);
    }

    const scan = async (dir, parentIgnore) => {
      if (hardStopFlag?.stopSearch) {
        return [];
      }

      const localIgnore = [...parentIgnore];

      if (!omitGitIgnore) {
        const gitignore = await getGitIgnoreContentForDirectory(dir);
        localIgnore.push(...gitignore);
      }

      const ignoreInstance = (0, _ignore.default)();
      ignoreInstance.add(localIgnore);
      const entriesList = await _fs.promises.readdir(dir, {// withFileTypes: true // This should work but throws an error, so we have to workaround
      });
      const absolutePaths = entriesList.map(entry => _path.default.join(dir, entry));
      const filteredAbsolutePaths = ignoreInstance // remove initial '/' or 'C:\' as ignore instance is not allowing for absolute paths
      .filter(absolutePaths.map(p => p.replace(fsRoot, ''))) // add initial '/' or 'C:\' back
      .map(p => `${fsRoot}${p}`);
      const absolutePathsWithStats = await Promise.all(filteredAbsolutePaths.map(async absolutePath => {
        try {
          let resolvedPath = absolutePath;
          let stat = await _fs.promises.lstat(absolutePath);

          if (stat.isSymbolicLink()) {
            resolvedPath = await _fs.promises.realpath(absolutePath);
            stat = await _fs.promises.stat(resolvedPath);
          }

          return {
            absolutePath,
            size: stat.size,
            isFile: stat.isFile(),
            isDirectory: stat.isDirectory()
          };
        } catch (e) {
          return {
            absolutePath,
            size: -1,
            isFile: false,
            isDirectory: false
          };
        }
      }));
      let directories = absolutePathsWithStats.filter(({
        isDirectory
      }) => isDirectory).map(({
        absolutePath
      }) => absolutePath);
      directories = directories.filter(pathName => {
        const isOnBlackList = directoriesBlackList.some(directoryName => pathName.endsWith(directoryName));
        return !isOnBlackList;
      });
      const files = absolutePathsWithStats.filter(({
        isFile,
        size
      }) => isFile && size < bigFileSizeInBytes || searchBigFiles).map(({
        absolutePath
      }) => absolutePath);
      const directoriesScanResult = (await Promise.all(directories.map(dir => scan(dir, localIgnore)))).flat();
      return [...files.filter(pathName => extensionTester.test(pathName)), ...directoriesScanResult];
    };

    filesList = await scan(searchRoot, InitialIgnore);
  }

  const filteredExcludeInclude = filterIncludeExclude({
    filesList,
    searchRoot,
    exclude,
    include
  });
  return filteredExcludeInclude;
};

exports.getFilesList = getFilesList;