@vidoc/scip-typescript
Version:
SCIP indexer for TypeScript and JavaScript
413 lines (412 loc) • 17.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.main = main;
exports.indexCommand = indexCommand;
const child_process = __importStar(require("child_process"));
const fs = __importStar(require("fs"));
// import { EOL } from 'os'
const path = __importStar(require("path"));
const url = __importStar(require("url"));
const ts = __importStar(require("typescript"));
const package_json_1 = __importDefault(require("../package.json"));
const CommandLineOptions_1 = require("./CommandLineOptions");
const inferTsconfig_1 = require("./inferTsconfig");
const ProjectIndexer_1 = require("./ProjectIndexer");
const scip = __importStar(require("./scip"));
function main() {
(0, CommandLineOptions_1.mainCommand)((projects, options) => indexCommand(projects, options)).parse(process.argv);
return;
}
const rgPath = path.resolve(require.resolve('@vscode/ripgrep'), '../../bin/rg');
const escapeQuery = (query) => {
return query.replace(/"/g, `\\"`).replace(/`/g, `\\\``);
};
const searchFilenames = async (query, cwd) => {
return new Promise((resolve, reject) => {
const p = child_process.spawn(`${rgPath} --files | ${rgPath} "${escapeQuery(query)}"`, [], { cwd, shell: true });
let output = '';
let error = '';
p.stdout.on('data', data => {
output += data.toString();
});
p.stderr.on('data', data => {
error += data.toString();
});
p.on('error', err => {
console.error('Error: ', { error: err });
reject(err);
});
p.on('close', () => {
const lines = output.split('\n').filter(line => line.length > 0);
resolve(lines);
if (error.length > 0) {
console.error('Error: ', { error });
}
});
});
};
// detect all subprojects in the workspace
// we use the package.json to detect all subprojects
// we ripgrep to do it
const findSubprojects = async (rootDir) => {
const files = await searchFilenames('package.json', rootDir);
// we only want to do exact match on package.json
const packageJsonFiles = files.filter(file => path.basename(file) === 'package.json');
return packageJsonFiles.map(file => path.join(rootDir, path.dirname(file)));
};
async function indexCommand(_projects, options) {
const root = options.cwd;
console.log('Root', root);
const projects = [];
// by default we want to detect all subprojects in the workspace
const subprojects = await findSubprojects(options.cwd);
console.log('Subprojects', subprojects, 'projects', projects);
subprojects.forEach(subproject => {
if (!projects.includes(subproject)) {
projects.push(subproject);
}
});
// if we have root in the projects, we need to move it at the end
if (projects.includes(root)) {
const projectIndex = projects.indexOf(root);
projects.push(projects.splice(projectIndex, 1)[0]);
}
// we need extra step to decide if we want to discard the root project
// if we have subprojects, and root dosn't have .js,.ts files that
// are not in the subprojects, we want to discard the root project
// we can do this by checking if the root project has any .js,.ts files
// that are not in the subprojects
const allFiles = await searchFilenames('(\.js|\.ts|\.jsx|\.tsx|\.mjs|\.cjs|\.mts|\.cts)$', root);
// check if all files in root are in subprojects
const rootFilesNotInSubprojects = allFiles
.map(file => path.join(root, file))
.filter(file => !projects.some(subproject => file.includes(subproject)));
console.log('rootFilesNotInSubprojects', rootFilesNotInSubprojects.length);
const subdirectoriesWithoutRoot = [...projects]
.map(project => path.relative(root, project))
.filter(directory => directory !== '');
// if for some reason we don't have any projects, we want to add the root project
if (projects.length === 0) {
projects.push(root);
}
console.log('Final projects', projects);
options.cwd = makeAbsolutePath(process.cwd(), options.cwd);
options.output = makeAbsolutePath(options.cwd, options.output);
if (!options.indexedProjects) {
options.indexedProjects = new Set();
}
const output = fs.openSync(options.output, 'w');
let documentCount = 0;
const writeIndex = (index) => {
documentCount += index.documents.length;
fs.writeSync(output, index.serializeBinary());
};
const cache = {
sources: new Map(),
parsedCommandLines: new Map(),
};
const indexedFiles = [];
try {
writeIndex(new scip.scip.Index({
metadata: new scip.scip.Metadata({
project_root: url.pathToFileURL(options.cwd).toString(),
text_document_encoding: scip.scip.TextEncoding.UTF8,
tool_info: new scip.scip.ToolInfo({
name: 'scip-typescript',
version: package_json_1.default.version,
arguments: [],
}),
}),
}));
// NOTE: we may want index these projects in parallel in the future.
// We need to be careful about which order we index the projects because
// they can have dependencies.
for (const projectRoot of projects) {
// when we are scanning the root of project, we want to exclude the subdirectories
// because we dont want to index the same files twice
const excludeDirs = projectRoot === root ? subdirectoriesWithoutRoot : [];
// console.log('excludeDirs', excludeDirs, projectRoot)
const projectDisplayName = projectRoot === '.' ? options.cwd : projectRoot;
const indexedFilesFromSingleProject = indexSingleProject({
...options,
projectRoot,
projectDisplayName,
writeIndex,
}, cache, excludeDirs,
// has to be absolute paths because we are using path.relative
allFiles
.map(file => path.join(root, file))
// only include files that are in the project root
.filter(file => file.includes(projectRoot)), indexedFiles, projectRoot === root);
// we need to clear sources from the cache that have node_modules in the path
for (const path of cache.sources.keys()) {
cache.sources.delete(path);
}
if (indexedFilesFromSingleProject) {
indexedFiles.push(...indexedFilesFromSingleProject);
}
}
}
finally {
fs.close(output);
if (documentCount > 0) {
console.log(`done ${options.output}`);
}
else {
process.exitCode = 1;
fs.rmSync(options.output);
const prettyProjects = JSON.stringify(projects);
console.log(`error: no files got indexed. To fix this problem, make sure that the TypeScript projects ${prettyProjects} contain input files or reference other projects.`);
}
}
}
function makeAbsolutePath(cwd, relativeOrAbsolutePath) {
if (path.isAbsolute(relativeOrAbsolutePath)) {
return relativeOrAbsolutePath;
}
return path.resolve(cwd, relativeOrAbsolutePath);
}
function indexSingleProject(options, cache, excludeDirs, allFiles, indexedFiles, isRootScan) {
if (options.indexedProjects.has(options.projectRoot)) {
return;
}
options.indexedProjects.add(options.projectRoot);
console.log('options.projectRoot', options.projectRoot);
let config = ts.parseCommandLine(['-p', options.projectRoot], (relativePath) => path.resolve(options.projectRoot, relativePath));
console.log('config', config);
// if we are scanning the root of the project
// we want to check if we are covering all possible files returned from config.fileNames
if (isRootScan && allFiles.length > 0) {
// console.log('config', config.fileNames)
const filesNotFound = allFiles.filter(file => !config.fileNames.includes(file) && !indexedFiles.includes(file));
console.log(`filesNotFound in ${options.projectRoot}`, filesNotFound);
// we want to add the root project to the list of projects
config.fileNames.push(...filesNotFound);
}
let tsconfigFileName;
if (config.options.project) {
const projectPath = path.resolve(config.options.project);
if (ts.sys.directoryExists(projectPath)) {
tsconfigFileName = path.join(projectPath, 'tsconfig.json');
}
else {
tsconfigFileName = projectPath;
}
const fileExists = ts.sys.fileExists(tsconfigFileName);
const loadedConfig = loadConfigFile(projectPath, fileExists
? {
file: tsconfigFileName,
}
: { jsonConfig: (0, inferTsconfig_1.inferTsconfig)(projectPath) }, excludeDirs);
if (loadedConfig !== undefined) {
config = loadedConfig;
// check if we are covering all possible files returned from config.fileNames
const filesNotFound = allFiles.filter(file => !config.fileNames.includes(file) && !indexedFiles.includes(file));
if (filesNotFound.length > 0) {
config.fileNames.push(...filesNotFound);
}
}
}
console.log('LOADED CONFIG', config);
for (const projectReference of config.projectReferences || []) {
// if projectReference is inside current project, we want to skip it
if (projectReference.path.includes(options.projectRoot)) {
continue;
}
indexSingleProject({
...options,
projectRoot: projectReference.path,
projectDisplayName: projectReference.path,
}, cache, excludeDirs, allFiles, [...config.fileNames, ...indexedFiles], false);
}
console.log(`config for ${options.projectRoot}`, config.fileNames.length);
// run indexer if there are files to index
if (config.fileNames.length > 0) {
new ProjectIndexer_1.ProjectIndexer(config, options, cache).index();
}
return [...config.fileNames];
}
if (require.main === module) {
main();
}
const readConfigFile = (file, absolute) => {
const readResult = ts.readConfigFile(absolute, path => ts.sys.readFile(path));
if (readResult.error) {
throw new Error(ts.formatDiagnostics([readResult.error], ts.createCompilerHost({})));
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const config = readResult.config;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (config.compilerOptions !== undefined) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
config.compilerOptions = {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
...config.compilerOptions,
...defaultCompilerOptions(file),
};
}
return config;
};
function loadConfigFile(basePath, opts, excludeDirs) {
let config;
if ('file' in opts) {
const absolute = path.resolve(opts.file);
config = readConfigFile(opts.file, absolute);
}
else {
config = opts.jsonConfig;
}
if (excludeDirs.length > 0) {
console.log('excludeDirs', excludeDirs);
config.exclude = [...(config.exclude || []), ...excludeDirs];
}
console.log('Just before parse');
const result = ts.parseJsonConfigFileContent(config, ts.sys, basePath);
console.log('result', result);
const errors = [];
for (const error of result.errors) {
if (error.code === 18003) {
// Ignore errors about missing 'input' fields, example:
// > TS18003: No inputs were found in config file 'tsconfig.json'. Specified 'include' paths were '[]' and 'exclude' paths were '["out","node_modules","dist"]'.
// The reason we ignore this error here is because we report the same
// error at a higher-level. It's common to hit on a single TypeScript
// project with no sources when using the --yarnWorkspaces option.
// Instead of failing fast at that single project, we only report this
// error if all projects have no files.
continue;
}
errors.push(error);
}
if (errors.length > 0) {
console.log(ts.formatDiagnostics(errors, ts.createCompilerHost({})));
// We know that the config is invalid, but we want to continue
// because we want to index the files that are in the project
return result;
}
return result;
}
function defaultCompilerOptions(configFileName) {
const options =
// Not a typo, jsconfig.json is a thing https://sourcegraph.com/search?q=context:global+file:jsconfig.json&patternType=literal
configFileName && path.basename(configFileName) === 'jsconfig.json'
? {
allowJs: true,
maxNodeModuleJsDepth: 2,
allowSyntheticDefaultImports: true,
skipLibCheck: true,
noEmit: true,
}
: {};
return options;
}
// function listPnpmWorkspaces(directory: string): string[] {
// /**
// * Returns the list of projects formatted as:
// * '/Users/user/sourcegraph/client/web:@sourcegraph/web@1.10.1:PRIVATE',
// *
// * See https://pnpm.io/id/cli/list#--depth-number
// */
// const output = child_process.execSync(
// 'pnpm ls -r --depth -1 --long --parseable',
// {
// cwd: directory,
// encoding: 'utf-8',
// maxBuffer: 1024 * 1024 * 5, // 5MB
// }
// )
// return output
// .split(EOL)
// .filter(project => project.includes(':'))
// .map(project => project.split(':')[0])
// }
// function listYarnWorkspaces(
// directory: string,
// yarnVersion: 'tryYarn1' | 'yarn2Plus'
// ): string[] {
// const runYarn = (cmd: string): string =>
// child_process.execSync(cmd, {
// cwd: directory,
// encoding: 'utf-8',
// maxBuffer: 1024 * 1024 * 5, // 5MB
// })
// const result: string[] = []
// const yarn1WorkspaceInfo = (): void => {
// // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
// const json = JSON.parse(
// // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
// JSON.parse(runYarn('yarn --silent --json workspaces info')).data
// )
// for (const key of Object.keys(json)) {
// const location = 'location'
// // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
// if (json[key][location] !== undefined) {
// // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
// result.push(path.join(directory, json[key][location]))
// }
// }
// }
// const yarn2PlusWorkspaceInfo = (): void => {
// const jsonLines = runYarn('yarn --json workspaces list').split(
// /\r?\n|\r|\n/g
// )
// for (let line of jsonLines) {
// line = line.trim()
// if (line.length === 0) {
// continue
// }
// // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
// const json = JSON.parse(line)
// if ('location' in json) {
// // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
// result.push(path.join(directory, json.location))
// }
// }
// }
// if (yarnVersion === 'tryYarn1') {
// try {
// yarn2PlusWorkspaceInfo()
// } catch {
// yarn1WorkspaceInfo()
// }
// } else {
// yarn2PlusWorkspaceInfo()
// }
// return result
// }