hikma-engine
Version:
Code Knowledge Graph Indexer - A sophisticated TypeScript-based indexer that transforms Git repositories into multi-dimensional knowledge stores for AI agents
622 lines (621 loc) • 25.9 kB
JavaScript
"use strict";
/**
* Enhanced AST Parser for deep code analysis
* Extracts Functions, Variables, Classes, Imports, Exports with relationships
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.EnhancedASTParser = void 0;
const ts = __importStar(require("typescript"));
const fs = __importStar(require("fs"));
const logger_1 = require("../utils/logger");
const enhanced_graph_1 = require("../types/enhanced-graph");
class EnhancedASTParser {
constructor() {
this.logger = (0, logger_1.getLogger)('EnhancedASTParser');
this.nodes = [];
this.edges = [];
this.currentFileId = '';
this.currentRepoId = '';
this.currentCommitSha = '';
this.sourceFile = null;
}
/**
* Parse a TypeScript/JavaScript file and extract enhanced AST information
*/
async parseFile(filePath, repoId, commitSha, content) {
this.logger.debug(`Parsing file: ${filePath}`);
// Reset state
this.nodes = [];
this.edges = [];
this.currentRepoId = repoId;
this.currentCommitSha = commitSha;
this.currentFileId = enhanced_graph_1.BusinessKeyGenerator.file(repoId, commitSha, filePath);
try {
// Read file content if not provided
const fileContent = content || fs.readFileSync(filePath, 'utf-8');
// Create TypeScript source file
this.sourceFile = ts.createSourceFile(filePath, fileContent, ts.ScriptTarget.Latest, true);
// Visit all nodes in the AST
this.visitNode(this.sourceFile);
this.logger.debug(`Parsed ${filePath}: ${this.nodes.length} nodes, ${this.edges.length} edges`);
return {
nodes: [...this.nodes],
edges: [...this.edges]
};
}
catch (error) {
this.logger.error(`Failed to parse ${filePath}`, { error });
return { nodes: [], edges: [] };
}
}
visitNode(node) {
switch (node.kind) {
case ts.SyntaxKind.FunctionDeclaration:
this.processFunctionDeclaration(node);
break;
case ts.SyntaxKind.ArrowFunction:
this.processArrowFunction(node);
break;
case ts.SyntaxKind.MethodDeclaration:
this.processMethodDeclaration(node);
break;
case ts.SyntaxKind.ClassDeclaration:
this.processClassDeclaration(node);
break;
case ts.SyntaxKind.VariableDeclaration:
this.processVariableDeclaration(node);
break;
case ts.SyntaxKind.ImportDeclaration:
this.processImportDeclaration(node);
break;
case ts.SyntaxKind.ExportDeclaration:
case ts.SyntaxKind.ExportAssignment:
this.processExportDeclaration(node);
break;
case ts.SyntaxKind.CallExpression:
this.processCallExpression(node);
break;
case ts.SyntaxKind.Identifier:
this.processIdentifier(node);
break;
}
// Continue visiting child nodes
ts.forEachChild(node, child => this.visitNode(child));
}
processFunctionDeclaration(node) {
if (!node.name)
return;
const functionName = node.name.text;
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const endPos = this.sourceFile.getLineAndCharacterOfPosition(node.getEnd());
const businessKey = enhanced_graph_1.BusinessKeyGenerator.function(this.currentFileId, functionName, startPos.line + 1);
const functionNode = {
id: businessKey,
businessKey,
type: 'Function',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: functionName,
async: !!(node.modifiers?.some(m => m.kind === ts.SyntaxKind.AsyncKeyword)),
generator: !!node.asteriskToken,
params: node.parameters.map(p => p.name?.getText() || '').filter(Boolean),
returnType: node.type?.getText(),
loc: endPos.line - startPos.line + 1,
startLine: startPos.line + 1,
endLine: endPos.line + 1,
body: node.body?.getText() || '',
docstring: this.extractJSDoc(node)
}
};
this.nodes.push(functionNode);
// Create DECLARES edge from file to function
this.edges.push({
id: `${this.currentFileId}-DECLARES-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'DECLARES',
line: startPos.line + 1,
col: startPos.character + 1
});
// Process function body for variable reads/writes and calls
if (node.body) {
this.processFunctionBody(node.body, businessKey);
}
}
processArrowFunction(node) {
const parent = node.parent;
let functionName = 'anonymous';
// Try to get function name from variable declaration or property assignment
if (ts.isVariableDeclaration(parent) && parent.name) {
functionName = parent.name.getText();
}
else if (ts.isPropertyAssignment(parent) && parent.name) {
functionName = parent.name.getText();
}
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const endPos = this.sourceFile.getLineAndCharacterOfPosition(node.getEnd());
const businessKey = enhanced_graph_1.BusinessKeyGenerator.function(this.currentFileId, functionName, startPos.line + 1);
const arrowFunctionNode = {
id: businessKey,
businessKey,
type: 'ArrowFunction',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: functionName,
async: !!(node.modifiers?.some(m => m.kind === ts.SyntaxKind.AsyncKeyword)),
generator: false,
params: node.parameters.map(p => p.name?.getText() || '').filter(Boolean),
returnType: node.type?.getText(),
loc: endPos.line - startPos.line + 1,
startLine: startPos.line + 1,
endLine: endPos.line + 1,
body: node.body.getText(),
docstring: this.extractJSDoc(node)
}
};
this.nodes.push(arrowFunctionNode);
// Create DECLARES edge
this.edges.push({
id: `${this.currentFileId}-DECLARES-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'DECLARES',
line: startPos.line + 1,
col: startPos.character + 1
});
// Process function body
this.processFunctionBody(node.body, businessKey);
}
processMethodDeclaration(node) {
if (!node.name)
return;
const methodName = node.name.getText();
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const endPos = this.sourceFile.getLineAndCharacterOfPosition(node.getEnd());
const businessKey = enhanced_graph_1.BusinessKeyGenerator.function(this.currentFileId, methodName, startPos.line + 1);
const methodNode = {
id: businessKey,
businessKey,
type: 'Function',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: methodName,
async: !!(node.modifiers?.some(m => m.kind === ts.SyntaxKind.AsyncKeyword)),
generator: !!node.asteriskToken,
params: node.parameters.map(p => p.name?.getText() || '').filter(Boolean),
returnType: node.type?.getText(),
loc: endPos.line - startPos.line + 1,
startLine: startPos.line + 1,
endLine: endPos.line + 1,
body: node.body?.getText() || '',
docstring: this.extractJSDoc(node)
}
};
this.nodes.push(methodNode);
// Create DECLARES edge
this.edges.push({
id: `${this.currentFileId}-DECLARES-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'DECLARES',
line: startPos.line + 1,
col: startPos.character + 1
});
// Process method body
if (node.body) {
this.processFunctionBody(node.body, businessKey);
}
}
processClassDeclaration(node) {
if (!node.name)
return;
const className = node.name.text;
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const endPos = this.sourceFile.getLineAndCharacterOfPosition(node.getEnd());
const businessKey = enhanced_graph_1.BusinessKeyGenerator.class(this.currentFileId, className);
const classNode = {
id: businessKey,
businessKey,
type: 'Class',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: className,
isAbstract: !!(node.modifiers?.some(m => m.kind === ts.SyntaxKind.AbstractKeyword)),
extends: node.heritageClauses?.find(h => h.token === ts.SyntaxKind.ExtendsKeyword)?.types[0]?.getText(),
implements: node.heritageClauses?.find(h => h.token === ts.SyntaxKind.ImplementsKeyword)?.types.map(t => t.getText()),
decorators: node.decorators?.map((d) => d.getText()),
startLine: startPos.line + 1,
endLine: endPos.line + 1
}
};
this.nodes.push(classNode);
// Create DECLARES edge
this.edges.push({
id: `${this.currentFileId}-DECLARES-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'DECLARES',
line: startPos.line + 1,
col: startPos.character + 1
});
// Process inheritance relationships
if (classNode.properties.extends) {
// Create EXTENDS edge (would need to resolve the target class)
// For now, we'll create a placeholder
this.edges.push({
id: `${businessKey}-EXTENDS-${classNode.properties.extends}`,
source: businessKey,
target: classNode.properties.extends,
sourceBusinessKey: businessKey,
targetBusinessKey: classNode.properties.extends,
type: 'EXTENDS'
});
}
}
processVariableDeclaration(node) {
if (!node.name || !ts.isIdentifier(node.name))
return;
const varName = node.name.text;
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const businessKey = enhanced_graph_1.BusinessKeyGenerator.variable(this.currentFileId, varName, startPos.line + 1);
// Determine variable kind
const parent = node.parent;
let kind = 'var';
if (ts.isVariableDeclarationList(parent)) {
if (parent.flags & ts.NodeFlags.Const)
kind = 'const';
else if (parent.flags & ts.NodeFlags.Let)
kind = 'let';
}
const variableNode = {
id: businessKey,
businessKey,
type: 'Variable',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: varName,
kind,
typeAnnotation: node.type?.getText(),
valueSnippet: node.initializer?.getText()?.substring(0, 100),
isExported: this.isExported(node),
scope: this.determineScope(node)
}
};
this.nodes.push(variableNode);
// Create DECLARES edge
this.edges.push({
id: `${this.currentFileId}-DECLARES-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'DECLARES',
line: startPos.line + 1,
col: startPos.character + 1
});
}
processImportDeclaration(node) {
if (!node.moduleSpecifier || !ts.isStringLiteral(node.moduleSpecifier))
return;
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const businessKey = enhanced_graph_1.BusinessKeyGenerator.import(this.currentFileId, startPos.line + 1);
const sourceModule = node.moduleSpecifier.text;
const importedNames = [];
const localNames = [];
let isDefault = false;
let isNamespace = false;
if (node.importClause) {
// Default import
if (node.importClause.name) {
isDefault = true;
importedNames.push('default');
localNames.push(node.importClause.name.text);
}
// Named imports
if (node.importClause.namedBindings) {
if (ts.isNamespaceImport(node.importClause.namedBindings)) {
isNamespace = true;
importedNames.push('*');
localNames.push(node.importClause.namedBindings.name.text);
}
else if (ts.isNamedImports(node.importClause.namedBindings)) {
node.importClause.namedBindings.elements.forEach(element => {
importedNames.push(element.propertyName?.text || element.name.text);
localNames.push(element.name.text);
});
}
}
}
const importNode = {
id: businessKey,
businessKey,
type: 'Import',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
isDefault,
isNamespace,
sourceModule,
importedNames,
localNames
}
};
this.nodes.push(importNode);
// Create IMPORTS edge from file to module
this.edges.push({
id: `${this.currentFileId}-IMPORTS-${sourceModule}`,
source: this.currentFileId,
target: sourceModule,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: sourceModule,
type: 'IMPORTS',
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
importedNames,
localNames,
isDefault,
isNamespace
}
});
}
processExportDeclaration(node) {
const startPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
if (ts.isExportDeclaration(node)) {
// Handle export { ... } from '...'
if (node.exportClause && ts.isNamedExports(node.exportClause)) {
node.exportClause.elements.forEach(element => {
const exportName = element.name.text;
const businessKey = enhanced_graph_1.BusinessKeyGenerator.export(this.currentFileId, exportName);
const exportNode = {
id: businessKey,
businessKey,
type: 'Export',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: exportName,
type: 'named',
isReExport: !!node.moduleSpecifier,
sourceModule: node.moduleSpecifier?.getText()
}
};
this.nodes.push(exportNode);
// Create EXPORTS edge
this.edges.push({
id: `${this.currentFileId}-EXPORTS-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'EXPORTS',
line: startPos.line + 1,
col: startPos.character + 1
});
});
}
}
else if (ts.isExportAssignment(node)) {
// Handle export = ... or export default ...
const businessKey = enhanced_graph_1.BusinessKeyGenerator.export(this.currentFileId, 'default');
const exportNode = {
id: businessKey,
businessKey,
type: 'Export',
repoId: this.currentRepoId,
commitSha: this.currentCommitSha,
filePath: this.sourceFile.fileName,
line: startPos.line + 1,
col: startPos.character + 1,
properties: {
name: 'default',
type: 'default',
isReExport: false
}
};
this.nodes.push(exportNode);
// Create EXPORTS edge
this.edges.push({
id: `${this.currentFileId}-EXPORTS-${businessKey}`,
source: this.currentFileId,
target: businessKey,
sourceBusinessKey: this.currentFileId,
targetBusinessKey: businessKey,
type: 'EXPORTS',
line: startPos.line + 1,
col: startPos.character + 1
});
}
}
processCallExpression(node) {
// This will be called from within function bodies to track function calls
// Implementation depends on the current function context
}
processIdentifier(node) {
// Track variable reads/writes within function contexts
// Implementation depends on the current function context
}
processFunctionBody(body, functionBusinessKey) {
// Recursively process function body to find:
// - Function calls (CALLS edges)
// - Variable reads (READS edges)
// - Variable writes (WRITES edges)
const processNode = (node) => {
if (ts.isCallExpression(node)) {
this.processFunctionCall(node, functionBusinessKey);
}
else if (ts.isIdentifier(node)) {
this.processVariableAccess(node, functionBusinessKey);
}
ts.forEachChild(node, processNode);
};
processNode(body);
}
processFunctionCall(node, callerBusinessKey) {
const callPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
// Try to resolve the called function
let calledFunctionName = '';
if (ts.isIdentifier(node.expression)) {
calledFunctionName = node.expression.text;
}
else if (ts.isPropertyAccessExpression(node.expression)) {
calledFunctionName = node.expression.name.text;
}
if (calledFunctionName) {
// Create a placeholder business key for the called function
// In a real implementation, you'd resolve this properly
const calleeBusinessKey = `${this.currentFileId}#${calledFunctionName}#unknown`;
this.edges.push({
id: `${callerBusinessKey}-CALLS-${calleeBusinessKey}`,
source: callerBusinessKey,
target: calleeBusinessKey,
sourceBusinessKey: callerBusinessKey,
targetBusinessKey: calleeBusinessKey,
type: 'CALLS',
line: callPos.line + 1,
col: callPos.character + 1,
dynamic: this.isDynamicCall(node)
});
}
}
processVariableAccess(node, functionBusinessKey) {
const accessPos = this.sourceFile.getLineAndCharacterOfPosition(node.getStart());
const varName = node.text;
// Determine if this is a read or write
const isWrite = this.isWriteAccess(node);
const edgeType = isWrite ? 'WRITES' : 'READS';
// Create placeholder variable business key
const varBusinessKey = enhanced_graph_1.BusinessKeyGenerator.variable(this.currentFileId, varName, accessPos.line + 1);
this.edges.push({
id: `${functionBusinessKey}-${edgeType}-${varBusinessKey}`,
source: functionBusinessKey,
target: varBusinessKey,
sourceBusinessKey: functionBusinessKey,
targetBusinessKey: varBusinessKey,
type: edgeType,
line: accessPos.line + 1,
col: accessPos.character + 1
});
}
// Helper methods
extractJSDoc(node) {
const jsDoc = node.jsDoc;
if (jsDoc && jsDoc.length > 0) {
return jsDoc[0].comment;
}
return undefined;
}
isExported(node) {
let current = node.parent;
while (current) {
if (ts.isExportDeclaration(current) || ts.isExportAssignment(current)) {
return true;
}
if (current.modifiers?.some((m) => m.kind === ts.SyntaxKind.ExportKeyword)) {
return true;
}
current = current.parent;
}
return false;
}
determineScope(node) {
let current = node.parent;
while (current) {
if (ts.isFunctionDeclaration(current) || ts.isArrowFunction(current) || ts.isMethodDeclaration(current)) {
return 'function';
}
if (ts.isBlock(current)) {
return 'block';
}
current = current.parent;
}
return 'global';
}
isDynamicCall(node) {
// Check if this is a dynamic call like require() or import()
if (ts.isIdentifier(node.expression)) {
return ['require', 'import'].includes(node.expression.text);
}
return false;
}
isWriteAccess(node) {
const parent = node.parent;
// Check if this identifier is on the left side of an assignment
if (ts.isBinaryExpression(parent) && parent.operatorToken.kind === ts.SyntaxKind.EqualsToken) {
return parent.left === node;
}
// Check for other write patterns (++, --, +=, etc.)
if (ts.isPostfixUnaryExpression(parent) || ts.isPrefixUnaryExpression(parent)) {
return [ts.SyntaxKind.PlusPlusToken, ts.SyntaxKind.MinusMinusToken].includes(parent.operator);
}
return false;
}
}
exports.EnhancedASTParser = EnhancedASTParser;