depsweep
Version:
🌱 Automated intelligent dependency cleanup with environmental impact reporting
411 lines (410 loc) • 15.8 kB
JavaScript
import { readdirSync } from "node:fs";
import * as fs from "node:fs/promises";
import path from "node:path";
import chalk from "chalk";
import { findUp } from "find-up";
import { globby } from "globby";
import { isBinaryFileSync } from "isbinaryfile";
import { FILE_PATTERNS, MESSAGES } from "./constants.js";
import { isConfigFile, parseConfigFile, isDependencyUsedInFile, } from "./helpers.js";
import { OptimizedCache, OptimizedFileReader, OptimizedDependencyAnalyzer, StringOptimizer, OptimizedFileSystem, PerformanceMonitor, MemoryOptimizer, } from "./performance-optimizations.js";
import { customSort } from "./index.js";
const depInfoCache = new OptimizedCache(2000, 300000);
const performanceMonitor = PerformanceMonitor.getInstance();
const memoryOptimizer = MemoryOptimizer.getInstance();
const fileReader = OptimizedFileReader.getInstance();
const dependencyAnalyzer = OptimizedDependencyAnalyzer.getInstance();
const fileSystem = OptimizedFileSystem.getInstance();
function normalizeTypesPackage(typesPackage) {
const basePackage = typesPackage.replace("@types/", "");
if (basePackage.includes("__")) {
return `@${basePackage.replace("__", "/")}`;
}
return basePackage.includes("/") ? `@${basePackage}` : basePackage;
}
function getFrameworkInfo(context) {
const packageJson = context.configs?.["package.json"];
if (!packageJson)
return null;
const deps = packageJson.dependencies || {};
const developmentDeps = packageJson.devDependencies || {};
const allDeps = { ...deps, ...developmentDeps };
const frameworks = [
{
name: "angular",
corePackage: "@angular/core",
devDependencies: [
"@angular-builders/",
"@angular-devkit/",
"@angular/cli",
"@webcomponents/custom-elements",
],
},
{
name: "react",
corePackage: "react",
devDependencies: [
"react-scripts",
"@testing-library/react",
"react-app-rewired",
],
},
];
for (const framework of frameworks) {
if (allDeps[framework.corePackage]) {
return framework;
}
}
return null;
}
function isFrameworkDevelopmentDependency(dependency, frameworkInfo) {
if (!frameworkInfo)
return false;
return frameworkInfo.devDependencies.some((prefix) => dependency.startsWith(prefix) || dependency === prefix);
}
export async function getDependencyInfo(dependency, context, sourceFiles, topLevelDependencies, progressOptions) {
performanceMonitor.startTimer("getDependencyInfo");
const memoryStats = memoryOptimizer.checkMemoryUsage();
if (memoryStats.shouldGC) {
depInfoCache.clear();
fileReader.clearCache();
dependencyAnalyzer.clearCaches();
}
const cacheKey = StringOptimizer.intern(`${context.projectRoot}:${dependency}`);
const cached = depInfoCache.get(cacheKey);
if (cached !== undefined) {
performanceMonitor.endTimer("getDependencyInfo");
return cached;
}
const info = {
usedInFiles: [],
requiredByPackages: new Set(),
hasSubDependencyUsage: false,
};
const frameworkInfo = getFrameworkInfo(context);
if (frameworkInfo &&
isFrameworkDevelopmentDependency(dependency, frameworkInfo)) {
info.requiredByPackages.add(frameworkInfo.corePackage);
return info;
}
if (dependency.startsWith("@types/")) {
const basePackage = normalizeTypesPackage(dependency);
const tsConfig = await getTSConfig(context.projectRoot);
if (basePackage === "node" && hasTSFiles(sourceFiles)) {
info.requiredByPackages.add("typescript");
return info;
}
if (topLevelDependencies.has(basePackage)) {
info.requiredByPackages.add(basePackage);
}
let subdepIndex = 0;
for (const file of sourceFiles) {
subdepIndex++;
if ((file.endsWith(".ts") || file.endsWith(".tsx")) &&
((await isDependencyUsedInFile(dependency, file, context)) ||
(await isDependencyUsedInFile(basePackage, file, context)))) {
info.usedInFiles.push(file);
}
progressOptions?.onProgress?.(file, subdepIndex);
await new Promise((res) => setImmediate(res));
}
if (tsConfig && hasTSFiles(sourceFiles)) {
const { types = [], typeRoots = [] } = tsConfig.compilerOptions || {};
if (types.includes(basePackage) ||
typeRoots.some((root) => root.includes(basePackage))) {
info.requiredByPackages.add("typescript");
}
}
return info;
}
const subdeps = context.dependencyGraph?.get(dependency) || new Set();
const subdepsArray = [...subdeps];
const totalSubdeps = subdepsArray.length;
performanceMonitor.startTimer("fileProcessing");
const usedFiles = await dependencyAnalyzer.processFilesInBatches(sourceFiles, dependency, context, (processed, total) => {
progressOptions?.onProgress?.(sourceFiles[processed - 1], processed, total);
});
info.usedInFiles = usedFiles;
if (subdepsArray.length > 0) {
for (const [index, subdep] of subdepsArray.entries()) {
const subdepUsedFiles = await dependencyAnalyzer.processFilesInBatches(sourceFiles, subdep, context);
if (subdepUsedFiles.length > 0) {
info.hasSubDependencyUsage = true;
break;
}
progressOptions?.onProgress?.(sourceFiles[0], index + 1, totalSubdeps);
}
}
performanceMonitor.endTimer("fileProcessing");
const nodeModulesPath = path.join(context.projectRoot, "node_modules");
try {
const packages = new Set();
const entries = readdirSync(nodeModulesPath, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory())
continue;
if (entry.name.startsWith("@")) {
const scopedDir = path.join(nodeModulesPath, entry.name);
const scopedEntries = readdirSync(scopedDir, { withFileTypes: true });
for (const sub of scopedEntries) {
if (sub.isDirectory()) {
packages.add(path.join(entry.name, sub.name));
}
}
}
else {
packages.add(entry.name);
}
}
performanceMonitor.startTimer("packageJsonReading");
const packageJsonPromises = [...packages].map(async (package_) => {
try {
const packagePath = StringOptimizer.intern(path.join(nodeModulesPath, package_, "package.json"));
const data = await fileReader.readFile(packagePath);
return {
pkg: StringOptimizer.intern(package_),
data: JSON.parse(data),
};
}
catch {
return null;
}
});
const packageJsonResults = await Promise.all(packageJsonPromises);
performanceMonitor.endTimer("packageJsonReading");
const dependencyGraph = new Map();
for (const result of packageJsonResults) {
if (!result)
continue;
const { pkg, data } = result;
const allDeps = {
...data.dependencies,
...data.peerDependencies,
...data.optionalDependencies,
};
dependencyGraph.set(pkg, new Set(Object.keys(allDeps)));
}
const findTopLevelDependents = (dep) => {
const dependents = new Set();
for (const [package_, deps] of dependencyGraph.entries()) {
if (deps.has(dep)) {
if (topLevelDependencies.has(package_)) {
dependents.add(package_);
}
else {
const parentDeps = findTopLevelDependents(package_);
for (const parentDep of parentDeps) {
dependents.add(parentDep);
}
}
}
}
return dependents;
};
const allRequiringPackages = findTopLevelDependents(dependency);
info.requiredByPackages = allRequiringPackages;
}
catch {
}
depInfoCache.set(cacheKey, info);
performanceMonitor.endTimer("getDependencyInfo");
return info;
}
export async function getTSConfig(projectRoot) {
try {
const tsConfigPath = path.join(projectRoot, "tsconfig.json");
const content = await fs.readFile(tsConfigPath, "utf8");
return JSON.parse(content);
}
catch {
return null;
}
}
function hasTSFiles(files) {
return files.some((file) => file.endsWith(".ts") || file.endsWith(".tsx"));
}
export async function getWorkspaceInfo(packageJsonPath) {
try {
const content = await fs.readFile(packageJsonPath);
const package_ = JSON.parse(content.toString("utf8"));
if (!package_.workspaces)
return undefined;
const patterns = Array.isArray(package_.workspaces)
? package_.workspaces
: package_.workspaces.packages || [];
const packagePaths = await globby(patterns, {
cwd: path.dirname(packageJsonPath),
onlyDirectories: true,
expandDirectories: false,
ignore: ["node_modules"],
});
return {
root: packageJsonPath,
packages: packagePaths,
};
}
catch {
return undefined;
}
}
export async function findClosestPackageJson(startDirectory) {
const packageJsonPath = await findUp(FILE_PATTERNS.PACKAGE_JSON, {
cwd: startDirectory,
});
if (!packageJsonPath) {
console.error(chalk.red(MESSAGES.noPackageJson));
process.exit(1);
}
let currentDirectory = path.dirname(packageJsonPath);
while (true) {
const parentDirectory = path.dirname(currentDirectory);
if (parentDirectory === currentDirectory) {
break;
}
const potentialRootPackageJson = path.join(parentDirectory, FILE_PATTERNS.PACKAGE_JSON);
try {
const rootPackageString = await fs.readFile(potentialRootPackageJson);
const rootPackage = JSON.parse(rootPackageString.toString("utf8"));
if (rootPackage.workspaces) {
console.log(chalk.yellow(MESSAGES.monorepoDetected));
return potentialRootPackageJson;
}
}
catch {
}
const workspaceInfo = await getWorkspaceInfo(potentialRootPackageJson);
if (workspaceInfo) {
const relativePath = path.relative(path.dirname(workspaceInfo.root), packageJsonPath);
const isWorkspacePackage = workspaceInfo.packages.some((p) => relativePath.startsWith(p) || p.startsWith(relativePath));
if (isWorkspacePackage) {
console.log(chalk.yellow("\nMonorepo workspace package detected."));
console.log(chalk.yellow(`Root: ${workspaceInfo.root}`));
return packageJsonPath;
}
}
currentDirectory = parentDirectory;
}
return packageJsonPath;
}
export async function getDependencies(packageJsonPath) {
const packageJsonString = (await fs.readFile(packageJsonPath, "utf8")) || "{}";
const packageJson = JSON.parse(packageJsonString);
const dependencies = packageJson.dependencies
? Object.keys(packageJson.dependencies)
: [];
const devDependencies = packageJson.devDependencies
? Object.keys(packageJson.devDependencies)
: [];
const peerDependencies = packageJson.peerDependencies
? Object.keys(packageJson.peerDependencies)
: [];
const optionalDependencies = packageJson.optionalDependencies
? Object.keys(packageJson.optionalDependencies)
: [];
const allDependencies = [
...dependencies,
...devDependencies,
...peerDependencies,
...optionalDependencies,
];
allDependencies.sort(customSort);
return allDependencies;
}
export async function getPackageContext(packageJsonPath) {
const projectDirectory = path.dirname(packageJsonPath);
const configs = {};
const dependencyGraph = new Map();
const dependencies = await getDependencies(packageJsonPath);
for (const dep of dependencies) {
dependencyGraph.set(dep, new Set());
}
const allFiles = await getSourceFiles(projectDirectory);
for (const file of allFiles) {
if (file && isConfigFile(file)) {
const relativePath = path.relative(projectDirectory, file);
try {
configs[relativePath] = await parseConfigFile(file);
}
catch {
}
}
}
const packageJsonString = (await fs.readFile(packageJsonPath, "utf8")) || "{}";
const packageJson = JSON.parse(packageJsonString);
return {
scripts: packageJson.scripts,
configs: {
"package.json": packageJson,
...configs,
},
projectRoot: path.dirname(packageJsonPath),
dependencyGraph,
};
}
export async function getSourceFiles(projectDirectory, ignorePatterns = []) {
const files = await globby(["**/*"], {
cwd: projectDirectory,
gitignore: true,
ignore: [
FILE_PATTERNS.NODE_MODULES,
"dist",
"coverage",
"build",
".git",
"*.log",
"*.lock",
...ignorePatterns,
],
absolute: true,
});
return files.filter((file) => !isBinaryFileSync(file));
}
export function scanForDependency(config, dependency) {
if (!config) {
return false;
}
if (Array.isArray(config)) {
return config.some((item) => {
if (typeof item === "string") {
return item.includes(dependency);
}
return scanForDependency(item, dependency);
});
}
if (typeof config !== "object") {
return false;
}
for (const [key, value] of Object.entries(config)) {
if (typeof value === "string" && value.includes(dependency)) {
return true;
}
if (typeof value === "object" && scanForDependency(value, dependency)) {
return true;
}
}
return false;
}
export async function processFilesInParallel(files, dependency, context, onProgress) {
performanceMonitor.startTimer("processFilesInParallel");
const memoryStats = memoryOptimizer.getMemoryStats();
const availableMemory = memoryStats.heapTotal - memoryStats.heapUsed;
const BATCH_SIZE = Math.min(200, Math.max(20, Math.floor(availableMemory / (1024 * 1024 * 25))));
const results = [];
let totalErrors = 0;
const usedFiles = await dependencyAnalyzer.processFilesInBatches(files, dependency, context, onProgress);
for (const file of usedFiles) {
if (file) {
results.push(file);
}
}
performanceMonitor.endTimer("processFilesInParallel");
if (totalErrors > 0) {
console.warn(chalk.yellow(`\nWarning: ${totalErrors} files had processing errors`));
}
return results;
}
export function findSubDependencies(dependency, context) {
return context.dependencyGraph?.get(dependency)
? [...context.dependencyGraph.get(dependency)]
: [];
}