sicua
Version:
A tool for analyzing project structure and dependencies
646 lines (645 loc) • 27.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.processFile = processFile;
exports.parseFiles = parseFiles;
exports.parsePackageJson = parsePackageJson;
// General Imports
const path = __importStar(require("path"));
const parser_1 = require("@babel/parser");
const traverse_1 = __importDefault(require("@babel/traverse"));
const t = __importStar(require("@babel/types"));
const worker_threads_1 = require("worker_threads");
const os = __importStar(require("os"));
const pathUtils_1 = require("../utils/common/pathUtils");
const reactSpecific_1 = require("../utils/ast/reactSpecific");
const configManager_1 = require("../core/configManager");
let errorCount = 0;
const MAX_ERROR_LOGS = 3;
const errorFiles = new Set();
/**
* Create parse context from config and scan result
*/
function createParseContext(config) {
const projectStructure = config.getProjectStructure();
return {
projectType: projectStructure?.projectType || "react",
routerType: projectStructure?.routerType,
sourceDirectory: config.srcDir,
projectRoot: config.projectPath,
};
}
/**
* Normalize file path relative to appropriate base directory
*/
function normalizeFilePath(filePath, context) {
// Try to make path relative to source directory first
if (filePath.startsWith(context.sourceDirectory)) {
const relativePath = path.relative(context.sourceDirectory, filePath);
return relativePath || ".";
}
// Fallback to project root
const relativePath = path.relative(context.projectRoot, filePath);
return relativePath || ".";
}
/**
* Extract directory for component relation
*/
function extractDirectory(filePath, context) {
const normalizedPath = normalizeFilePath(filePath, context);
const directory = path.dirname(normalizedPath);
// Handle root directory cases
if (directory === "." || directory === "") {
return "/";
}
return directory;
}
/**
* Resolve import paths considering project structure
*/
function resolveImportPath(importPath, currentFile, context) {
// Skip external packages
if (!importPath.startsWith(".") && !importPath.startsWith("/")) {
return importPath;
}
try {
const currentDir = path.dirname(currentFile);
let resolvedPath;
if (importPath.startsWith(".")) {
// Relative import
resolvedPath = path.resolve(currentDir, importPath);
}
else {
// Absolute import from project root
resolvedPath = path.resolve(context.projectRoot, importPath.substring(1));
}
// Normalize the resolved path
return normalizeFilePath(resolvedPath, context);
}
catch (error) {
console.warn(`Failed to resolve import path: ${importPath} from ${currentFile}`);
return importPath;
}
}
/**
* Enhanced component detection based on project structure - but analyze ALL files for SAST
*/
function shouldTreatAsComponent(filePath, context) {
const fileName = path.basename(filePath, path.extname(filePath));
const normalizedPath = normalizeFilePath(filePath, context);
// Next.js specific component detection
if (context.projectType === "nextjs") {
// App router specific files
if (context.routerType === "app") {
const appRouterFiles = [
"layout",
"page",
"loading",
"error",
"not-found",
"template",
"default",
];
if (appRouterFiles.includes(fileName.toLowerCase())) {
return true;
}
}
// Pages router specific files
if (context.routerType === "pages") {
const pagesRouterFiles = ["_app", "_document", "_error", "404", "500"];
if (pagesRouterFiles.includes(fileName)) {
return true;
}
}
}
// General component detection - but still analyze ALL files for security
const isReactFile = filePath.endsWith(".tsx") || filePath.endsWith(".jsx");
const isComponentName = /^[A-Z]/.test(fileName); // Starts with capital letter
const isInComponentsDir = normalizedPath.includes("component");
// For SAST purposes, we want to analyze all files, but identify what's likely a component
return isReactFile && (isComponentName || isInComponentsDir);
}
/**
* Parse file content using Babel AST for component analysis with enhanced project structure awareness
*/
function parseFileContent(content, filePath, context) {
const analysis = {
imports: [],
lazyImports: [],
hocConnections: [],
exports: [],
components: new Map(),
globalFunctions: [],
globalFunctionCalls: {},
};
try {
const ast = (0, parser_1.parse)(content, {
sourceType: "module",
plugins: [
"jsx",
"typescript",
"decorators",
"classProperties",
"exportDefaultFrom",
"exportNamespaceFrom",
"dynamicImport",
],
errorRecovery: true,
});
let currentFunction;
let currentComponent;
(0, traverse_1.default)(ast, {
ImportDeclaration(path) {
const importPath = path.node.source.value;
const resolvedPath = resolveImportPath(importPath, filePath, context);
analysis.imports.push(resolvedPath);
},
CallExpression(path) {
const callName = getCallExpressionName(path.node);
// Handle lazy imports with enhanced resolution
if (callName === "lazy" || callName === "React.lazy") {
const arg = path.node.arguments[0];
if (t.isArrowFunctionExpression(arg) || t.isFunctionExpression(arg)) {
const body = arg.body;
if (t.isCallExpression(body) && t.isImport(body.callee)) {
const importArg = body.arguments[0];
if (t.isStringLiteral(importArg)) {
const resolvedPath = resolveImportPath(importArg.value, filePath, context);
analysis.lazyImports.push(resolvedPath);
}
}
}
}
// Track function calls for components
if (callName && currentComponent) {
const component = analysis.components.get(currentComponent);
if (component) {
if (!component.functionCalls[currentFunction || currentComponent]) {
component.functionCalls[currentFunction || currentComponent] = [];
}
component.functionCalls[currentFunction || currentComponent].push(callName);
}
}
else if (callName && currentFunction) {
if (!analysis.globalFunctionCalls[currentFunction]) {
analysis.globalFunctionCalls[currentFunction] = [];
}
analysis.globalFunctionCalls[currentFunction].push(callName);
}
},
ExportNamedDeclaration(path) {
if (path.node.specifiers) {
path.node.specifiers.forEach((specifier) => {
if (t.isExportSpecifier(specifier)) {
if (t.isIdentifier(specifier.exported)) {
analysis.exports.push(specifier.exported.name);
}
}
});
}
if (path.node.source) {
const resolvedPath = resolveImportPath(path.node.source.value, filePath, context);
analysis.imports.push(resolvedPath);
}
// Handle exported function/class declarations
if (path.node.declaration) {
const declaration = path.node.declaration;
if (t.isFunctionDeclaration(declaration) && declaration.id) {
const functionName = declaration.id.name;
analysis.exports.push(functionName);
if ((0, reactSpecific_1.isReactComponentBabel)(declaration, functionName) ||
shouldTreatAsComponent(filePath, context)) {
const componentInfo = createComponentInfo(functionName, false, true);
analysis.components.set(functionName, componentInfo);
}
}
if (t.isVariableDeclaration(declaration)) {
declaration.declarations.forEach((declarator) => {
if (t.isIdentifier(declarator.id) && declarator.init) {
const varName = declarator.id.name;
analysis.exports.push(varName);
if ((t.isArrowFunctionExpression(declarator.init) ||
t.isFunctionExpression(declarator.init)) &&
((0, reactSpecific_1.isReactComponentBabel)(declarator.init, varName) ||
shouldTreatAsComponent(filePath, context))) {
const componentInfo = createComponentInfo(varName, false, true);
analysis.components.set(varName, componentInfo);
}
}
});
}
}
},
ExportDefaultDeclaration(path) {
if (t.isCallExpression(path.node.declaration)) {
const callExpr = path.node.declaration;
if (t.isIdentifier(callExpr.callee) &&
callExpr.callee.name === "compose") {
callExpr.arguments.forEach((arg) => {
if (t.isIdentifier(arg)) {
analysis.hocConnections.push(arg.name);
}
else if (t.isCallExpression(arg) &&
t.isIdentifier(arg.callee)) {
analysis.hocConnections.push(arg.callee.name);
}
});
}
}
// Handle default exported components with project structure awareness
if (t.isFunctionDeclaration(path.node.declaration) &&
path.node.declaration.id) {
const functionName = path.node.declaration.id.name;
if ((0, reactSpecific_1.isReactComponentBabel)(path.node.declaration, functionName) ||
shouldTreatAsComponent(filePath, context)) {
const componentInfo = createComponentInfo(functionName, true, true);
analysis.components.set(functionName, componentInfo);
}
}
if (t.isIdentifier(path.node.declaration)) {
const componentName = path.node.declaration.name;
if (analysis.components.has(componentName)) {
const component = analysis.components.get(componentName);
component.isDefault = true;
}
else if (shouldTreatAsComponent(filePath, context)) {
// Create component info for files that should be treated as components
const componentInfo = createComponentInfo(componentName, true, true);
analysis.components.set(componentName, componentInfo);
}
}
},
FunctionDeclaration: {
enter(path) {
if (path.node.id && t.isIdentifier(path.node.id)) {
const functionName = path.node.id.name;
if ((0, reactSpecific_1.isReactComponentBabel)(path.node, functionName) ||
shouldTreatAsComponent(filePath, context)) {
currentComponent = functionName;
if (!analysis.components.has(functionName)) {
const componentInfo = createComponentInfo(functionName, false, false);
analysis.components.set(functionName, componentInfo);
}
}
else {
analysis.globalFunctions.push(functionName);
}
currentFunction = functionName;
}
},
exit() {
currentFunction = undefined;
currentComponent = undefined;
},
},
ArrowFunctionExpression: {
enter(path) {
const functionName = (0, reactSpecific_1.getBabelFunctionName)(path.node, path.parent);
if (functionName &&
((0, reactSpecific_1.isReactComponentBabel)(path.node, functionName) ||
shouldTreatAsComponent(filePath, context))) {
currentComponent = functionName;
if (!analysis.components.has(functionName)) {
const componentInfo = createComponentInfo(functionName, false, false);
analysis.components.set(functionName, componentInfo);
}
}
else if (functionName) {
analysis.globalFunctions.push(functionName);
}
currentFunction = functionName || currentFunction;
},
exit() {
currentFunction = undefined;
currentComponent = undefined;
},
},
VariableDeclaration(path) {
path.node.declarations.forEach((declaration) => {
if (t.isIdentifier(declaration.id) && declaration.init) {
const varName = declaration.id.name;
if ((t.isArrowFunctionExpression(declaration.init) ||
t.isFunctionExpression(declaration.init)) &&
((0, reactSpecific_1.isReactComponentBabel)(declaration.init, varName) ||
shouldTreatAsComponent(filePath, context))) {
if (!analysis.components.has(varName)) {
const componentInfo = createComponentInfo(varName, false, false);
analysis.components.set(varName, componentInfo);
}
}
}
});
},
});
}
catch (error) {
console.error(`Error parsing AST for ${filePath}:`, error);
return analysis;
}
return analysis;
}
/**
* Create component info structure
*/
function createComponentInfo(name, isDefault, isExported) {
return {
name,
isDefault,
isExported,
functions: [name],
functionCalls: {},
};
}
/**
* Get call expression name helper
*/
function getCallExpressionName(node) {
if (t.isIdentifier(node.callee)) {
return node.callee.name;
}
else if (t.isMemberExpression(node.callee)) {
if (t.isIdentifier(node.callee.object) &&
t.isIdentifier(node.callee.property)) {
return `${node.callee.object.name}.${node.callee.property.name}`;
}
else if (t.isIdentifier(node.callee.property)) {
return node.callee.property.name;
}
}
return "";
}
/**
* Process a single file and return multiple ComponentRelations with enhanced path handling
*/
async function processFile(filePath, srcPath, config, scanResult) {
try {
const content = scanResult.fileContents.get(filePath) || "";
const context = createParseContext(config);
const directory = extractDirectory(filePath, context);
const { imports, lazyImports, hocConnections, exports, components, globalFunctions, globalFunctionCalls, } = parseFileContent(content, filePath, context);
const componentRelations = [];
// Create ComponentRelation for each detected component
for (const [componentName, componentInfo] of components) {
const usedBy = [...imports, ...lazyImports, ...hocConnections]
.map((imp) => {
// Handle both relative and absolute imports
if (imp.startsWith(".")) {
return path.basename(imp, path.extname(imp));
}
return path.basename(imp, path.extname(imp));
})
.filter((imp) => imp !== componentName);
const componentRelation = {
name: componentName,
usedBy,
directory,
imports,
exports,
fullPath: filePath,
functions: componentInfo.functions,
functionCalls: componentInfo.functionCalls,
content,
};
componentRelations.push(componentRelation);
}
// Always create fallback relation for SAST analysis - analyze ALL files
if (componentRelations.length === 0) {
const fileName = path.basename(filePath, path.extname(filePath));
const usedBy = [...imports, ...lazyImports, ...hocConnections]
.map((imp) => {
if (imp.startsWith(".")) {
return path.basename(imp, path.extname(imp));
}
return path.basename(imp, path.extname(imp));
})
.filter((imp) => imp !== fileName);
const fallbackRelation = {
name: fileName,
usedBy,
directory,
imports,
exports,
fullPath: filePath,
functions: globalFunctions,
functionCalls: globalFunctionCalls,
content,
};
componentRelations.push(fallbackRelation);
}
// Handle usedBy relationships between components in the same file
componentRelations.forEach((relation) => {
const componentsInSameFile = componentRelations
.filter((r) => r.name !== relation.name)
.map((r) => r.name);
relation.usedBy = [...relation.usedBy, ...componentsInSameFile];
});
return componentRelations;
}
catch (error) {
errorFiles.add(filePath);
if (errorCount < MAX_ERROR_LOGS) {
console.error(`Error processing file ${filePath}:`, error);
errorCount++;
}
else if (errorCount === MAX_ERROR_LOGS) {
console.warn(`Additional errors occurred. Suppressing further error messages...`);
errorCount++;
}
throw error;
}
}
/**
* Parse files using the unified scan data with enhanced project structure support
*/
async function parseFiles(scanResult, srcPath, config) {
if (worker_threads_1.isMainThread) {
errorCount = 0;
errorFiles.clear();
const filePaths = scanResult.filePaths;
// Use workers for parallel processing
return await processWithWorkers(filePaths, srcPath, config, scanResult);
}
else {
// Worker thread - process the chunk
const { chunk, srcPath, config: configData, fileContents, fileMetadata, } = worker_threads_1.workerData;
// Reconstruct config manager in worker
const config = new configManager_1.ConfigManager(configData.projectPath);
Object.assign(config, configData);
const workerScanResult = {
filePaths: chunk,
sourceFiles: new Map(),
fileContents: new Map(Object.entries(fileContents)),
fileMetadata: new Map(Object.entries(fileMetadata)),
securityFiles: [],
configFiles: [],
environmentFiles: [],
apiRoutes: [],
middlewareFiles: [],
packageInfo: [],
securityScanMetadata: {
scanTimestamp: Date.now(),
scanDuration: 0,
filesScanned: chunk.length,
securityIssuesFound: 0,
riskLevel: "low",
coveragePercentage: 0,
},
};
const results = await Promise.all(chunk.map(async (filePath) => {
try {
return await processFile(filePath, srcPath, config, workerScanResult);
}
catch (error) {
console.warn(`Worker failed to process ${filePath}:`, error);
return [];
}
}));
const flatResults = results.flat();
worker_threads_1.parentPort?.postMessage(flatResults);
return [];
}
}
/**
* Process files with worker threads for parallel processing
*/
async function processWithWorkers(filePaths, srcPath, config, scanResult) {
const numWorkers = Math.min(os.cpus().length, 6);
const chunkSize = Math.ceil(filePaths.length / numWorkers);
const chunks = chunkArray(filePaths, chunkSize);
try {
const workers = chunks.map((chunk, index) => new worker_threads_1.Worker(__filename, {
workerData: {
chunk,
srcPath,
config: {
projectPath: config.projectPath,
srcDir: config.srcDir,
fileExtensions: config.fileExtensions,
rootComponentNames: config.rootComponentNames,
outputFileName: config.outputFileName,
// Include project structure info for workers
_projectStructure: config.getProjectStructure(),
},
fileContents: Object.fromEntries(Array.from(scanResult.fileContents.entries()).filter(([path]) => chunk.includes(path))),
fileMetadata: Object.fromEntries(Array.from(scanResult.fileMetadata.entries()).filter(([path]) => chunk.includes(path))),
},
}));
const results = await Promise.all(workers.map((worker, index) => new Promise((resolve) => {
const timeout = setTimeout(() => {
console.warn(`Worker ${index} timeout, terminating...`);
worker.terminate();
resolve([]);
}, 120000);
worker.on("message", (result) => {
clearTimeout(timeout);
resolve(result);
});
worker.on("error", (error) => {
clearTimeout(timeout);
console.error(`Worker ${index} error:`, error);
resolve([]);
});
worker.on("exit", (code) => {
clearTimeout(timeout);
if (code !== 0) {
console.error(`Worker ${index} exited with code ${code}`);
}
resolve([]);
});
})));
return results.flat();
}
catch (error) {
console.error("Fatal error during worker processing:", error);
return [];
}
}
/**
* Split an array into chunks
*/
function chunkArray(array, size) {
return Array.from({ length: Math.ceil(array.length / size) }, (_, i) => array.slice(i * size, i * size + size));
}
// Worker thread entry point
if (!worker_threads_1.isMainThread) {
const { chunk, srcPath, config: configData, fileContents, fileMetadata, } = worker_threads_1.workerData;
// Reconstruct config manager in worker
const config = new configManager_1.ConfigManager(configData.projectPath);
Object.assign(config, configData);
const workerScanResult = {
filePaths: chunk,
sourceFiles: new Map(),
fileContents: new Map(Object.entries(fileContents)),
fileMetadata: new Map(Object.entries(fileMetadata)),
securityFiles: [],
configFiles: [],
environmentFiles: [],
apiRoutes: [],
middlewareFiles: [],
packageInfo: [],
securityScanMetadata: {
scanTimestamp: Date.now(),
scanDuration: 0,
filesScanned: chunk.length,
securityIssuesFound: 0,
riskLevel: "low",
coveragePercentage: 0,
},
};
Promise.all(chunk.map(async (filePath) => {
try {
return await processFile(filePath, srcPath, config, workerScanResult);
}
catch (error) {
console.warn(`Worker error processing ${filePath}:`, error);
return [];
}
}))
.then((results) => {
const flatResults = results.flat();
worker_threads_1.parentPort?.postMessage(flatResults);
})
.catch((error) => {
console.error("Worker fatal error:", error);
worker_threads_1.parentPort?.postMessage([]);
});
}
/**
* Parse package.json to extract dependencies
*/
async function parsePackageJson(projectPath) {
try {
const packageJsonPath = path.join(projectPath, "package.json");
const packageJson = await (0, pathUtils_1.readJsonFile)(packageJsonPath);
return packageJson.dependencies || {};
}
catch (error) {
console.error("Error parsing package.json:", error);
return {};
}
}