@vidoc/scip-typescript/src/main.ts

SCIP indexer for TypeScript and JavaScript

#!/usr/bin/env node
import * as child_process from 'child_process'
import * as fs from 'fs'
// import { EOL } from 'os'
import * as path from 'path'
import * as url from 'url'

import * as ts from 'typescript'

import packageJson from '../package.json'

import {
  GlobalCache,
  mainCommand,
  MultiProjectOptions,
  ProjectOptions,
} from './CommandLineOptions'
import { inferTsconfig } from './inferTsconfig'
import { ProjectIndexer } from './ProjectIndexer'
import * as scip from './scip'

export function main(): void {
  mainCommand((projects, options) => indexCommand(projects, options)).parse(
    process.argv
  )
  return
}

const rgPath = path.resolve(require.resolve('@vscode/ripgrep'), '../../bin/rg')

const escapeQuery = (query: string): string => {
  return query.replace(/"/g, `\\"`).replace(/`/g, `\\\``)
}

const searchFilenames = async (
  query: string,
  cwd: string
): Promise<string[]> => {
  return new Promise((resolve, reject) => {
    const p = child_process.spawn(
      `${rgPath} --files | ${rgPath} "${escapeQuery(query)}"`,
      [],
      { cwd, shell: true }
    )

    let output = ''
    let error = ''

    p.stdout.on('data', data => {
      output += data.toString()
    })
    p.stderr.on('data', data => {
      error += data.toString()
    })

    p.on('error', err => {
      console.error('Error: ', { error: err })
      reject(err)
    })

    p.on('close', () => {
      const lines = output.split('\n').filter(line => line.length > 0)
      resolve(lines)

      if (error.length > 0) {
        console.error('Error: ', { error })
      }
    })
  })
}

// detect all subprojects in the workspace
// we use the package.json to detect all subprojects
// we ripgrep to do it
const findSubprojects = async (rootDir: string): Promise<string[]> => {
  const files = await searchFilenames('package.json', rootDir)

  // we only want to do exact match on package.json
  const packageJsonFiles = files.filter(
    file => path.basename(file) === 'package.json'
  )

  return packageJsonFiles.map(file => path.join(rootDir, path.dirname(file)))
}

export async function indexCommand(
  _projects: string[],
  options: MultiProjectOptions
): Promise<void> {
  const root = options.cwd
  console.log('Root', root)

  const projects: string[] = []

  // by default we want to detect all subprojects in the workspace
  const subprojects = await findSubprojects(options.cwd)

  console.log('Subprojects', subprojects, 'projects', projects)

  subprojects.forEach(subproject => {
    if (!projects.includes(subproject)) {
      projects.push(subproject)
    }
  })

  // if we have root in the projects, we need to move it at the end
  if (projects.includes(root)) {
    const projectIndex = projects.indexOf(root)
    projects.push(projects.splice(projectIndex, 1)[0])
  }

  // we need extra step to decide if we want to discard the root project
  // if we have subprojects, and root dosn't have .js,.ts files that
  // are not in the subprojects, we want to discard the root project
  // we can do this by checking if the root project has any .js,.ts files
  // that are not in the subprojects
  const allFiles = await searchFilenames(
    '(\.js|\.ts|\.jsx|\.tsx|\.mjs|\.cjs|\.mts|\.cts)$',
    root
  )

  // check if all files in root are in subprojects
  const rootFilesNotInSubprojects = allFiles
    .map(file => path.join(root, file))
    .filter(file => !projects.some(subproject => file.includes(subproject)))

  console.log('rootFilesNotInSubprojects', rootFilesNotInSubprojects.length)

  const subdirectoriesWithoutRoot = [...projects]
    .map(project => path.relative(root, project))
    .filter(directory => directory !== '')

  // if for some reason we don't have any projects, we want to add the root project
  if (projects.length === 0) {
    projects.push(root)
  }

  console.log('Final projects', projects)

  options.cwd = makeAbsolutePath(process.cwd(), options.cwd)
  options.output = makeAbsolutePath(options.cwd, options.output)
  if (!options.indexedProjects) {
    options.indexedProjects = new Set()
  }
  const output = fs.openSync(options.output, 'w')
  let documentCount = 0
  const writeIndex = (index: scip.scip.Index): void => {
    documentCount += index.documents.length
    fs.writeSync(output, index.serializeBinary())
  }

  const cache: GlobalCache = {
    sources: new Map(),
    parsedCommandLines: new Map(),
  }

  const indexedFiles: string[] = []
  try {
    writeIndex(
      new scip.scip.Index({
        metadata: new scip.scip.Metadata({
          project_root: url.pathToFileURL(options.cwd).toString(),
          text_document_encoding: scip.scip.TextEncoding.UTF8,
          tool_info: new scip.scip.ToolInfo({
            name: 'scip-typescript',
            version: packageJson.version,
            arguments: [],
          }),
        }),
      })
    )
    // NOTE: we may want index these projects in parallel in the future.
    // We need to be careful about which order we index the projects because
    // they can have dependencies.
    for (const projectRoot of projects) {
      // when we are scanning the root of project, we want to exclude the subdirectories
      // because we dont want to index the same files twice
      const excludeDirs = projectRoot === root ? subdirectoriesWithoutRoot : []
      // console.log('excludeDirs', excludeDirs, projectRoot)

      const projectDisplayName = projectRoot === '.' ? options.cwd : projectRoot
      const indexedFilesFromSingleProject = indexSingleProject(
        {
          ...options,
          projectRoot,
          projectDisplayName,
          writeIndex,
        },
        cache,
        excludeDirs,
        // has to be absolute paths because we are using path.relative
        allFiles
          .map(file => path.join(root, file))
          // only include files that are in the project root
          .filter(file => file.includes(projectRoot)),
        indexedFiles,
        projectRoot === root
      )

      // we need to clear sources from the cache that have node_modules in the path
      for (const path of cache.sources.keys()) {
        cache.sources.delete(path)
      }

      if (indexedFilesFromSingleProject) {
        indexedFiles.push(...indexedFilesFromSingleProject)
      }
    }
  } finally {
    fs.close(output)
    if (documentCount > 0) {
      console.log(`done ${options.output}`)
    } else {
      process.exitCode = 1
      fs.rmSync(options.output)
      const prettyProjects = JSON.stringify(projects)
      console.log(
        `error: no files got indexed. To fix this problem, make sure that the TypeScript projects ${prettyProjects} contain input files or reference other projects.`
      )
    }
  }
}

function makeAbsolutePath(cwd: string, relativeOrAbsolutePath: string): string {
  if (path.isAbsolute(relativeOrAbsolutePath)) {
    return relativeOrAbsolutePath
  }
  return path.resolve(cwd, relativeOrAbsolutePath)
}

function indexSingleProject(
  options: ProjectOptions,
  cache: GlobalCache,
  excludeDirs: string[],
  allFiles: string[],
  indexedFiles: string[],
  isRootScan: boolean
) {
  if (options.indexedProjects.has(options.projectRoot)) {
    return
  }

  options.indexedProjects.add(options.projectRoot)
  console.log('options.projectRoot', options.projectRoot)
  let config = ts.parseCommandLine(
    ['-p', options.projectRoot],
    (relativePath: string) => path.resolve(options.projectRoot, relativePath)
  )

  console.log('config', config)

  // if we are scanning the root of the project
  // we want to check if we are covering all possible files returned from config.fileNames
  if (isRootScan && allFiles.length > 0) {
    // console.log('config', config.fileNames)

    const filesNotFound = allFiles.filter(
      file => !config.fileNames.includes(file) && !indexedFiles.includes(file)
    )

    console.log(`filesNotFound in ${options.projectRoot}`, filesNotFound)
    // we want to add the root project to the list of projects
    config.fileNames.push(...filesNotFound)
  }

  let tsconfigFileName: string | undefined
  if (config.options.project) {
    const projectPath = path.resolve(config.options.project)
    if (ts.sys.directoryExists(projectPath)) {
      tsconfigFileName = path.join(projectPath, 'tsconfig.json')
    } else {
      tsconfigFileName = projectPath
    }

    const fileExists = ts.sys.fileExists(tsconfigFileName)

    const loadedConfig = loadConfigFile(
      projectPath,
      fileExists
        ? {
            file: tsconfigFileName,
          }
        : { jsonConfig: inferTsconfig(projectPath) },
      excludeDirs
    )

    if (loadedConfig !== undefined) {
      config = loadedConfig

      // check if we are covering all possible files returned from config.fileNames
      const filesNotFound = allFiles.filter(
        file => !config.fileNames.includes(file) && !indexedFiles.includes(file)
      )

      if (filesNotFound.length > 0) {
        config.fileNames.push(...filesNotFound)
      }
    }
  }

  console.log('LOADED CONFIG', config)

  for (const projectReference of config.projectReferences || []) {
    // if projectReference is inside current project, we want to skip it
    if (projectReference.path.includes(options.projectRoot)) {
      continue
    }

    indexSingleProject(
      {
        ...options,
        projectRoot: projectReference.path,
        projectDisplayName: projectReference.path,
      },
      cache,
      excludeDirs,
      allFiles,
      [...config.fileNames, ...indexedFiles],
      false
    )
  }

  console.log(`config for ${options.projectRoot}`, config.fileNames.length)

  // run indexer if there are files to index
  if (config.fileNames.length > 0) {
    new ProjectIndexer(config, options, cache).index()
  }

  return [...config.fileNames]
}

if (require.main === module) {
  main()
}

const readConfigFile = (
  file: string,
  absolute: string
): ts.ParsedCommandLine | undefined => {
  const readResult = ts.readConfigFile(absolute, path => ts.sys.readFile(path))

  if (readResult.error) {
    throw new Error(
      ts.formatDiagnostics([readResult.error], ts.createCompilerHost({}))
    )
  }
  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
  const config = readResult.config
  // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
  if (config.compilerOptions !== undefined) {
    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
    config.compilerOptions = {
      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
      ...config.compilerOptions,
      ...defaultCompilerOptions(file),
    }
  }
  return config
}

function loadConfigFile(
  basePath: string,
  opts: { file: string } | { jsonConfig: any },
  excludeDirs: string[]
): ts.ParsedCommandLine | undefined {
  let config: any

  if ('file' in opts) {
    const absolute = path.resolve(opts.file)

    config = readConfigFile(opts.file, absolute)
  } else {
    config = opts.jsonConfig
  }

  if (excludeDirs.length > 0) {
    console.log('excludeDirs', excludeDirs)
    config.exclude = [...(config.exclude || []), ...excludeDirs]
  }

  console.log('Just before parse')
  const result = ts.parseJsonConfigFileContent(config, ts.sys, basePath)
  console.log('result', result)
  const errors: ts.Diagnostic[] = []
  for (const error of result.errors) {
    if (error.code === 18003) {
      // Ignore errors about missing 'input' fields, example:
      // > TS18003: No inputs were found in config file 'tsconfig.json'. Specified 'include' paths were '[]' and 'exclude' paths were '["out","node_modules","dist"]'.
      // The reason we ignore this error here is because we report the same
      // error at a higher-level.  It's common to hit on a single TypeScript
      // project with no sources when using the --yarnWorkspaces option.
      // Instead of failing fast at that single project, we only report this
      // error if all projects have no files.
      continue
    }
    errors.push(error)
  }
  if (errors.length > 0) {
    console.log(ts.formatDiagnostics(errors, ts.createCompilerHost({})))

    // We know that the config is invalid, but we want to continue
    // because we want to index the files that are in the project
    return result
  }
  return result
}

function defaultCompilerOptions(configFileName?: string): ts.CompilerOptions {
  const options: ts.CompilerOptions =
    // Not a typo, jsconfig.json is a thing https://sourcegraph.com/search?q=context:global+file:jsconfig.json&patternType=literal
    configFileName && path.basename(configFileName) === 'jsconfig.json'
      ? {
          allowJs: true,
          maxNodeModuleJsDepth: 2,
          allowSyntheticDefaultImports: true,
          skipLibCheck: true,
          noEmit: true,
        }
      : {}
  return options
}

// function listPnpmWorkspaces(directory: string): string[] {
//   /**
//    * Returns the list of projects formatted as:
//    * '/Users/user/sourcegraph/client/web:@sourcegraph/web@1.10.1:PRIVATE',
//    *
//    * See https://pnpm.io/id/cli/list#--depth-number
//    */
//   const output = child_process.execSync(
//     'pnpm ls -r --depth -1 --long --parseable',
//     {
//       cwd: directory,
//       encoding: 'utf-8',
//       maxBuffer: 1024 * 1024 * 5, // 5MB
//     }
//   )

//   return output
//     .split(EOL)
//     .filter(project => project.includes(':'))
//     .map(project => project.split(':')[0])
// }

// function listYarnWorkspaces(
//   directory: string,
//   yarnVersion: 'tryYarn1' | 'yarn2Plus'
// ): string[] {
//   const runYarn = (cmd: string): string =>
//     child_process.execSync(cmd, {
//       cwd: directory,
//       encoding: 'utf-8',
//       maxBuffer: 1024 * 1024 * 5, // 5MB
//     })
//   const result: string[] = []
//   const yarn1WorkspaceInfo = (): void => {
//     // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
//     const json = JSON.parse(
//       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
//       JSON.parse(runYarn('yarn --silent --json workspaces info')).data
//     )
//     for (const key of Object.keys(json)) {
//       const location = 'location'
//       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
//       if (json[key][location] !== undefined) {
//         // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
//         result.push(path.join(directory, json[key][location]))
//       }
//     }
//   }
//   const yarn2PlusWorkspaceInfo = (): void => {
//     const jsonLines = runYarn('yarn --json workspaces list').split(
//       /\r?\n|\r|\n/g
//     )
//     for (let line of jsonLines) {
//       line = line.trim()
//       if (line.length === 0) {
//         continue
//       }
//       // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
//       const json = JSON.parse(line)
//       if ('location' in json) {
//         // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
//         result.push(path.join(directory, json.location))
//       }
//     }
//   }
//   if (yarnVersion === 'tryYarn1') {
//     try {
//       yarn2PlusWorkspaceInfo()
//     } catch {
//       yarn1WorkspaceInfo()
//     }
//   } else {
//     yarn2PlusWorkspaceInfo()
//   }
//   return result
// }