@er77/code-graph-rag-mcp
Version:
Multi-agent LiteRAG MCP server for advanced code graph analysis
1,393 lines (1,369 loc) • 2.07 MB
JavaScript
#!/usr/bin/env node
import { createHash, randomUUID } from 'crypto';
import { EventEmitter } from 'events';
import { nanoid } from 'nanoid';
import fs2, { existsSync, mkdirSync, statSync, renameSync, readdirSync, unlinkSync, writeFileSync, readFileSync } from 'fs';
import path, { resolve, normalize, join, extname, dirname } from 'path';
import os, { homedir } from 'os';
import Database from 'better-sqlite3';
import { LRUCache } from 'lru-cache';
import { Readable, Transform, Writable } from 'stream';
import { pipeline } from 'stream/promises';
import url from 'url';
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js';
import { z } from 'zod';
import { zodToJsonSchema } from 'zod-to-json-schema';
import { parse as parse$1 } from 'yaml';
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
}) : x)(function(x) {
if (typeof require !== "undefined") return require.apply(this, arguments);
throw Error('Dynamic require of "' + x + '" is not supported');
});
var __glob = (map) => (path3) => {
var fn = map[path3];
if (fn) return fn();
throw new Error("Module not found in bundle: " + path3);
};
var __esm = (fn, res) => function __init() {
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
};
var __commonJS = (cb, mod2) => function __require2() {
return mod2 || (0, cb[__getOwnPropNames(cb)[0]])((mod2 = { exports: {} }).exports, mod2), mod2.exports;
};
var __export = (target, all) => {
for (var name2 in all)
__defProp(target, name2, { get: all[name2], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod2, isNodeMode, target) => (target = mod2 != null ? __create(__getProtoOf(mod2)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
__defProp(target, "default", { value: mod2, enumerable: true }) ,
mod2
));
var __toCommonJS = (mod2) => __copyProps(__defProp({}, "__esModule", { value: true }), mod2);
// src/types/agent.ts
var init_agent = __esm({
"src/types/agent.ts"() {
}
});
var BaseAgent;
var init_base = __esm({
"src/agents/base.ts"() {
init_agent();
BaseAgent = class extends EventEmitter {
id;
type;
status;
capabilities;
taskQueue = [];
currentTask = null;
metrics;
memoryUsage = 0;
cpuUsage = 0;
constructor(type, capabilities) {
super();
this.id = `${type}-${randomUUID().slice(0, 8)}`;
this.type = type;
this.status = "idle" /* IDLE */;
this.capabilities = capabilities;
this.metrics = {
agentId: this.id,
tasksProcessed: 0,
tasksSucceeded: 0,
tasksFailed: 0,
averageProcessingTime: 0,
currentMemoryMB: 0,
currentCpuPercent: 0,
lastActivity: Date.now()
};
this.startResourceMonitoring();
}
async initialize() {
console.log(`[${this.id}] Initializing agent...`);
this.status = "idle" /* IDLE */;
await this.onInitialize();
this.emit("initialized", this.id);
}
async shutdown() {
console.log(`[${this.id}] Shutting down agent...`);
this.status = "shutdown" /* SHUTDOWN */;
await this.onShutdown();
this.emit("shutdown", this.id);
}
canHandle(task) {
if (this.status !== "idle" /* IDLE */) return false;
if (this.taskQueue.length >= this.capabilities.maxConcurrency) return false;
if (this.memoryUsage > this.capabilities.memoryLimit * 0.9) return false;
return this.canProcessTask(task);
}
async process(task) {
if (!this.canHandle(task)) {
throw new Error(`Agent ${this.id} cannot handle task ${task.id}`);
}
this.taskQueue.push(task);
this.status = "busy" /* BUSY */;
this.currentTask = task;
task.startedAt = Date.now();
try {
const result = await this.processTask(task);
task.completedAt = Date.now();
task.result = result;
this.metrics.tasksProcessed++;
this.metrics.tasksSucceeded++;
this.updateAverageProcessingTime(task.completedAt - task.startedAt);
this.emit("task:completed", { agentId: this.id, task });
return result;
} catch (error) {
task.error = error;
task.completedAt = Date.now();
this.metrics.tasksProcessed++;
this.metrics.tasksFailed++;
this.emit("task:failed", { agentId: this.id, task, error });
throw error;
} finally {
this.taskQueue = this.taskQueue.filter((t) => t.id !== task.id);
this.currentTask = null;
if (this.taskQueue.length === 0) {
this.status = "idle" /* IDLE */;
}
this.metrics.lastActivity = Date.now();
}
}
async send(message) {
this.emit("message:send", message);
}
async receive(message) {
this.emit("message:received", message);
await this.handleMessage(message);
}
getMemoryUsage() {
return this.memoryUsage;
}
getCpuUsage() {
return this.cpuUsage;
}
getTaskQueue() {
return [...this.taskQueue];
}
getMetrics() {
return { ...this.metrics };
}
// Resource monitoring
startResourceMonitoring() {
setInterval(() => {
this.updateResourceUsage();
}, 1e3);
}
updateResourceUsage() {
const memUsed = process.memoryUsage();
this.memoryUsage = Math.round(memUsed.heapUsed / 1024 / 1024);
this.metrics.currentMemoryMB = this.memoryUsage;
this.cpuUsage = this.status === "busy" /* BUSY */ ? 50 : 5;
this.metrics.currentCpuPercent = this.cpuUsage;
}
updateAverageProcessingTime(duration) {
const prev = this.metrics.averageProcessingTime;
const count = this.metrics.tasksSucceeded;
this.metrics.averageProcessingTime = (prev * (count - 1) + duration) / count;
}
};
}
});
var KnowledgeBus, knowledgeBus;
var init_knowledge_bus = __esm({
"src/core/knowledge-bus.ts"() {
KnowledgeBus = class extends EventEmitter {
knowledge = /* @__PURE__ */ new Map();
subscriptions = /* @__PURE__ */ new Map();
messageQueue = [];
maxQueueSize = 1e3;
maxKnowledgePerTopic = 100;
constructor() {
super();
this.startCleanupInterval();
}
/**
* Publish knowledge to a topic
*/
publish(topic, data, source, ttl) {
const entry = {
id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
topic,
data,
source,
timestamp: Date.now(),
ttl
};
if (!this.knowledge.has(topic)) {
this.knowledge.set(topic, []);
}
const entries = this.knowledge.get(topic);
entries.push(entry);
if (entries.length > this.maxKnowledgePerTopic) {
entries.shift();
}
this.notifySubscribers(entry);
this.emit("knowledge:published", entry);
}
/**
* Subscribe to knowledge updates
*/
subscribe(agentId, topic, handler) {
const subscription = {
id: `sub-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
agentId,
topic,
handler
};
const topicKey = topic instanceof RegExp ? "*" : topic;
if (!this.subscriptions.has(topicKey)) {
this.subscriptions.set(topicKey, []);
}
this.subscriptions.get(topicKey).push(subscription);
this.emit("subscription:created", subscription);
return subscription.id;
}
/**
* Unsubscribe from knowledge updates
*/
unsubscribe(subscriptionId) {
for (const [, subs] of this.subscriptions) {
const index = subs.findIndex((s) => s.id === subscriptionId);
if (index !== -1) {
subs.splice(index, 1);
this.emit("subscription:removed", subscriptionId);
break;
}
}
}
/**
* Query existing knowledge
*/
query(topic, limit = 10) {
const results = [];
for (const [storedTopic, entries] of this.knowledge) {
if (this.matchesTopic(storedTopic, topic)) {
results.push(...entries);
}
}
return results.sort((a, b) => b.timestamp - a.timestamp).slice(0, limit);
}
/**
* Send a direct message between agents
*/
async sendMessage(message) {
if (this.messageQueue.length >= this.maxQueueSize) {
this.messageQueue.shift();
}
this.messageQueue.push(message);
this.emit("message:sent", message);
if (message.to === "*") {
this.publish(`message:${message.type}`, message.payload, message.from);
}
}
/**
* Get recent messages
*/
getRecentMessages(limit = 10) {
return this.messageQueue.slice(-limit);
}
/**
* Clear knowledge for a specific topic
*/
clearTopic(topic) {
this.knowledge.delete(topic);
this.emit("topic:cleared", topic);
}
/**
* Get statistics about the knowledge bus
*/
getStats() {
let entryCount = 0;
let subscriptionCount = 0;
for (const entries of this.knowledge.values()) {
entryCount += entries.length;
}
for (const subs of this.subscriptions.values()) {
subscriptionCount += subs.length;
}
return {
topicCount: this.knowledge.size,
entryCount,
subscriptionCount,
messageQueueSize: this.messageQueue.length
};
}
// Private methods
matchesTopic(storedTopic, pattern) {
if (typeof pattern === "string") {
if (pattern.includes("*")) {
const regex = new RegExp("^" + pattern.replace(/\*/g, ".*") + "$");
return regex.test(storedTopic);
}
return storedTopic === pattern;
}
return pattern.test(storedTopic);
}
notifySubscribers(entry) {
const exactSubs = this.subscriptions.get(entry.topic) || [];
for (const sub of exactSubs) {
this.callHandler(sub, entry);
}
const wildcardSubs = this.subscriptions.get("*") || [];
for (const sub of wildcardSubs) {
if (this.matchesTopic(entry.topic, sub.topic)) {
this.callHandler(sub, entry);
}
}
}
async callHandler(subscription, entry) {
try {
await subscription.handler(entry);
} catch (error) {
console.error(`Error in subscription handler for agent ${subscription.agentId}:`, error);
this.emit("subscription:error", { subscription, error });
}
}
startCleanupInterval() {
setInterval(() => {
this.cleanupExpiredKnowledge();
}, 6e4);
}
cleanupExpiredKnowledge() {
const now = Date.now();
let cleanedCount = 0;
for (const [topic, entries] of this.knowledge) {
const validEntries = entries.filter((entry) => {
if (entry.ttl && now - entry.timestamp > entry.ttl) {
cleanedCount++;
return false;
}
return true;
});
if (validEntries.length === 0) {
this.knowledge.delete(topic);
} else {
this.knowledge.set(topic, validEntries);
}
}
if (cleanedCount > 0) {
this.emit("cleanup:completed", { entriesRemoved: cleanedCount });
}
}
};
knowledgeBus = new KnowledgeBus();
}
});
var ID_LENGTH, DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT, MAX_SUBGRAPH_DEPTH, GraphStorageImpl;
var init_graph_storage = __esm({
"src/storage/graph-storage.ts"() {
ID_LENGTH = 12;
DEFAULT_QUERY_LIMIT = 100;
MAX_QUERY_LIMIT = 1e3;
MAX_SUBGRAPH_DEPTH = 5;
GraphStorageImpl = class {
constructor(sqliteManager) {
this.sqliteManager = sqliteManager;
this.db = sqliteManager.getConnection();
this.prepareStatements();
}
db;
// Prepared statements for performance
statements = {};
/**
* Prepare frequently used statements
*/
prepareStatements() {
this.statements.insertEntity = this.db.prepare(`
INSERT OR REPLACE INTO entities
(id, name, type, file_path, location, metadata, hash, created_at, updated_at,
complexity_score, language, size_bytes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
this.statements.updateEntity = this.db.prepare(`
UPDATE entities
SET name = ?, type = ?, location = ?, metadata = ?, hash = ?, updated_at = ?,
complexity_score = ?, language = ?, size_bytes = ?
WHERE id = ?
`);
this.statements.deleteEntity = this.db.prepare(`
DELETE FROM entities WHERE id = ?
`);
this.statements.getEntity = this.db.prepare(`
SELECT * FROM entities WHERE id = ?
`);
this.statements.insertRelationship = this.db.prepare(`
INSERT OR REPLACE INTO relationships
(id, from_id, to_id, type, metadata, weight, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
`);
this.statements.deleteRelationship = this.db.prepare(`
DELETE FROM relationships WHERE id = ?
`);
this.statements.updateFile = this.db.prepare(`
INSERT OR REPLACE INTO files
(path, hash, last_indexed, entity_count)
VALUES (?, ?, ?, ?)
`);
this.statements.getFile = this.db.prepare(`
SELECT * FROM files WHERE path = ?
`);
this.statements.insertPerformanceMetric = this.db.prepare(`
INSERT INTO performance_metrics (id, operation, duration_ms, entity_count, memory_usage, created_at)
VALUES (?, ?, ?, ?, ?, ?)
`);
}
// =============================================================================
// 4. ENTITY OPERATIONS
// =============================================================================
async insertEntity(entity) {
return this.measureOperation(
"insert_entity",
() => {
const now = Date.now();
const id = entity.id || this.generateId();
const complexityScore = entity.complexityScore ?? this.calculateComplexity(entity);
const language = entity.language ?? this.detectLanguage(entity.filePath);
const sizeBytes = entity.sizeBytes ?? 0;
this.statements.insertEntity.run(
id,
entity.name,
entity.type,
entity.filePath,
JSON.stringify(entity.location),
JSON.stringify(entity.metadata),
entity.hash,
entity.createdAt || now,
entity.updatedAt || now,
complexityScore,
language,
sizeBytes
);
},
1
);
}
async insertEntities(entities) {
return this.measureOperation(
"insert_entities_batch",
() => {
const start = Date.now();
const errors = [];
let processed = 0;
const transaction = this.db.transaction((entities2) => {
for (const entity of entities2) {
try {
const now = Date.now();
const id = entity.id || this.generateId();
const complexityScore = entity.complexityScore ?? this.calculateComplexity(entity);
const language = entity.language ?? this.detectLanguage(entity.filePath);
const sizeBytes = entity.sizeBytes ?? 0;
this.statements.insertEntity.run(
id,
entity.name,
entity.type,
entity.filePath,
JSON.stringify(entity.location),
JSON.stringify(entity.metadata),
entity.hash,
entity.createdAt || now,
entity.updatedAt || now,
complexityScore,
language,
sizeBytes
);
processed++;
} catch (error) {
errors.push({
item: entity,
error: error instanceof Error ? error.message : String(error)
});
}
}
});
try {
transaction(entities);
} catch (error) {
console.error("[GraphStorage] Batch insert failed:", error);
}
return {
processed,
failed: errors.length,
errors,
timeMs: Date.now() - start
};
},
entities.length
);
}
async updateEntity(id, updates) {
const existing = await this.getEntity(id);
if (!existing) {
throw new Error(`Entity ${id} not found`);
}
const updated = { ...existing, ...updates, updatedAt: Date.now() };
const complexityScore = updated.complexityScore ?? this.calculateComplexity(updated);
const language = updated.language ?? this.detectLanguage(updated.filePath);
const sizeBytes = updated.sizeBytes ?? 0;
this.statements.updateEntity.run(
updated.name,
updated.type,
JSON.stringify(updated.location),
JSON.stringify(updated.metadata),
updated.hash,
updated.updatedAt,
complexityScore,
language,
sizeBytes,
id
);
}
async deleteEntity(id) {
this.statements.deleteEntity.run(id);
}
async getEntity(id) {
const row = this.statements.getEntity.get(id);
return row ? this.rowToEntity(row) : null;
}
async findEntities(query) {
let sql = "SELECT * FROM entities WHERE 1=1";
const params = [];
if (query.filters) {
if (query.filters.entityType) {
const types = Array.isArray(query.filters.entityType) ? query.filters.entityType : [query.filters.entityType];
sql += ` AND type IN (${types.map(() => "?").join(",")})`;
params.push(...types);
}
if (query.filters.filePath) {
const paths = Array.isArray(query.filters.filePath) ? query.filters.filePath : [query.filters.filePath];
sql += ` AND file_path IN (${paths.map(() => "?").join(",")})`;
params.push(...paths);
}
if (query.filters.name) {
if (query.filters.name instanceof RegExp) {
const pattern = query.filters.name.source.replace(/\*/g, "%");
sql += " AND name LIKE ?";
params.push(pattern);
} else {
sql += " AND name = ?";
params.push(query.filters.name);
}
}
}
const limit = Math.min(query.limit || DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT);
sql += " LIMIT ? OFFSET ?";
params.push(limit, query.offset || 0);
const rows = this.db.prepare(sql).all(...params);
return rows.map((row) => this.rowToEntity(row));
}
// =============================================================================
// 5. RELATIONSHIP OPERATIONS
// =============================================================================
async insertRelationship(relationship) {
const id = relationship.id || this.generateId();
const now = Date.now();
this.statements.insertRelationship.run(
id,
relationship.fromId,
relationship.toId,
relationship.type,
relationship.metadata ? JSON.stringify(relationship.metadata) : null,
relationship.weight ?? 1,
relationship.createdAt ?? now
);
}
async insertRelationships(relationships) {
const start = Date.now();
const errors = [];
let processed = 0;
const transaction = this.db.transaction((relationships2) => {
for (const relationship of relationships2) {
try {
const id = relationship.id || this.generateId();
const now = Date.now();
this.statements.insertRelationship.run(
id,
relationship.fromId,
relationship.toId,
relationship.type,
relationship.metadata ? JSON.stringify(relationship.metadata) : null,
relationship.weight ?? 1,
relationship.createdAt ?? now
);
processed++;
} catch (error) {
errors.push({
item: relationship,
error: error instanceof Error ? error.message : String(error)
});
}
}
});
try {
transaction(relationships);
} catch (error) {
console.error("[GraphStorage] Batch relationship insert failed:", error);
}
return {
processed,
failed: errors.length,
errors,
timeMs: Date.now() - start
};
}
async deleteRelationship(id) {
this.statements.deleteRelationship.run(id);
}
async getRelationshipsForEntity(entityId, type) {
let sql = `
SELECT * FROM relationships
WHERE (from_id = ? OR to_id = ?)
`;
const params = [entityId, entityId];
if (type) {
sql += " AND type = ?";
params.push(type);
}
const rows = this.db.prepare(sql).all(...params);
return rows.map((row) => this.rowToRelationship(row));
}
async findRelationships(query) {
let sql = "SELECT * FROM relationships WHERE 1=1";
const params = [];
if (query.filters) {
if (query.filters.relationshipType) {
const types = Array.isArray(query.filters.relationshipType) ? query.filters.relationshipType : [query.filters.relationshipType];
sql += ` AND type IN (${types.map(() => "?").join(",")})`;
params.push(...types);
}
}
const limit = Math.min(query.limit || DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT);
sql += " LIMIT ? OFFSET ?";
params.push(limit, query.offset || 0);
const rows = this.db.prepare(sql).all(...params);
return rows.map((row) => this.rowToRelationship(row));
}
// =============================================================================
// 6. FILE OPERATIONS
// =============================================================================
async updateFileInfo(info) {
this.statements.updateFile.run(info.path, info.hash, info.lastIndexed, info.entityCount);
}
async getFileInfo(path3) {
const row = this.statements.getFile.get(path3);
return row ? {
path: row.path,
hash: row.hash,
lastIndexed: row.last_indexed,
entityCount: row.entity_count
} : null;
}
async getOutdatedFiles(since) {
const rows = this.db.prepare(`
SELECT * FROM files
WHERE last_indexed < ?
ORDER BY last_indexed ASC
`).all(since);
return rows.map((row) => ({
path: row.path,
hash: row.hash,
lastIndexed: row.last_indexed,
entityCount: row.entity_count
}));
}
// =============================================================================
// 7. QUERY OPERATIONS
// =============================================================================
async executeQuery(query) {
return this.measureOperation(
"execute_query",
async () => {
const start = Date.now();
const entities = await this.findEntities(query);
const relationships = await this.findRelationships(query);
const totalEntities = this.db.prepare("SELECT COUNT(*) as count FROM entities").get();
const totalRelationships = this.db.prepare("SELECT COUNT(*) as count FROM relationships").get();
return {
entities,
relationships,
stats: {
totalEntities: totalEntities.count,
totalRelationships: totalRelationships.count,
queryTimeMs: Date.now() - start
}
};
},
void 0
);
}
async getSubgraph(entityId, depth) {
return this.measureOperation(
"get_subgraph",
async () => {
const start = Date.now();
const maxDepth = Math.min(depth, MAX_SUBGRAPH_DEPTH);
const entities = /* @__PURE__ */ new Map();
const relationships = /* @__PURE__ */ new Map();
const visited = /* @__PURE__ */ new Set();
const queue = [{ id: entityId, level: 0 }];
while (queue.length > 0) {
const { id, level } = queue.shift();
if (visited.has(id) || level > maxDepth) continue;
visited.add(id);
const entity = await this.getEntity(id);
if (entity) {
entities.set(id, entity);
const rels = await this.getRelationshipsForEntity(id);
for (const rel of rels) {
relationships.set(rel.id, rel);
if (level < maxDepth) {
const nextId = rel.fromId === id ? rel.toId : rel.fromId;
if (!visited.has(nextId)) {
queue.push({ id: nextId, level: level + 1 });
}
}
}
}
}
return {
entities: Array.from(entities.values()),
relationships: Array.from(relationships.values()),
stats: {
totalEntities: entities.size,
totalRelationships: relationships.size,
queryTimeMs: Date.now() - start
}
};
},
1
);
}
// =============================================================================
// 8. MAINTENANCE OPERATIONS
// =============================================================================
async vacuum() {
this.sqliteManager.vacuum();
}
async analyze() {
this.sqliteManager.analyze();
}
async getMetrics() {
return this.measureOperation("get_metrics", async () => {
const baseMetrics = await this.sqliteManager.getMetrics();
const cacheStats = this.db.prepare(`
SELECT
COUNT(*) as entries,
SUM(hit_count) as hits,
SUM(miss_count) as misses
FROM query_cache
WHERE expires_at > ?
`).get(Date.now());
const embeddingsCount = this.db.prepare(`
SELECT COUNT(*) as count FROM embeddings
`).get();
const perfMetricsCount = this.db.prepare(`
SELECT COUNT(*) as count FROM performance_metrics
`).get();
const avgQueryTime = this.db.prepare(`
SELECT AVG(duration_ms) as avg_time
FROM performance_metrics
WHERE operation LIKE '%query%' AND created_at > ?
`).get(Date.now() - 24 * 60 * 60 * 1e3);
const lastVacuum = this.db.pragma("user_version", { simple: true });
let vectorSearchEnabled = false;
try {
this.db.prepare("SELECT vec_version()").get();
vectorSearchEnabled = true;
} catch {
}
const memoryUsage = process.memoryUsage();
return {
...baseMetrics,
cacheHitRate: cacheStats.hits + cacheStats.misses > 0 ? cacheStats.hits / (cacheStats.hits + cacheStats.misses) : 0,
lastVacuum,
// Enhanced v2 metrics
totalEmbeddings: embeddingsCount.count,
vectorSearchEnabled,
performanceMetricsCount: perfMetricsCount.count,
memoryUsageMB: Math.round(memoryUsage.heapUsed / 1024 / 1024),
concurrentConnections: 1,
// Single connection for now
averageQueryTimeMs: avgQueryTime.avg_time || 0
};
});
}
// =============================================================================
// 9. UTILITY METHODS
// =============================================================================
/**
* Generate unique ID
*/
generateId() {
return nanoid(ID_LENGTH);
}
/**
* Calculate complexity score for an entity
*/
calculateComplexity(entity) {
let score = 1;
switch (entity.type) {
case "function":
score = 2;
break;
case "class":
score = 3;
break;
case "method":
score = 2;
break;
case "interface":
score = 2;
break;
default:
score = 1;
}
if (entity.metadata.parameters?.length) {
score += Math.min(entity.metadata.parameters.length * 0.5, 3);
}
if (entity.metadata.modifiers?.length) {
score += Math.min(entity.metadata.modifiers.length * 0.3, 2);
}
return Math.round(score);
}
/**
* Record performance metric
*/
recordPerformanceMetric(operation, durationMs, entityCount, memoryUsage) {
try {
const id = this.generateId();
const now = Date.now();
this.statements.insertPerformanceMetric?.run(id, operation, durationMs, entityCount ?? 0, memoryUsage ?? 0, now);
} catch (error) {
console.warn("[GraphStorage] Performance metric recording failed:", error);
}
}
/**
* Measure operation performance
*/
async measureOperation(operation, fn, entityCount) {
const start = Date.now();
const startMemory = process.memoryUsage().heapUsed;
try {
const result = await fn();
const duration = Date.now() - start;
const memoryDelta = process.memoryUsage().heapUsed - startMemory;
this.recordPerformanceMetric(operation, duration, entityCount, memoryDelta);
return result;
} catch (error) {
const duration = Date.now() - start;
this.recordPerformanceMetric(`${operation}_error`, duration, entityCount);
throw error;
}
}
/**
* Detect programming language from file path
*/
detectLanguage(filePath) {
const ext = filePath.split(".").pop()?.toLowerCase();
switch (ext) {
case "ts":
case "tsx":
return "typescript";
case "js":
case "jsx":
case "mjs":
return "javascript";
case "py":
return "python";
case "java":
return "java";
case "c":
return "c";
case "cpp":
case "cc":
case "cxx":
return "cpp";
case "rs":
return "rust";
case "go":
return "go";
case "php":
return "php";
case "rb":
return "ruby";
case "swift":
return "swift";
case "kt":
return "kotlin";
default:
return "unknown";
}
}
/**
* Convert database row to Entity (enhanced for v2)
*/
rowToEntity(row) {
return {
id: row.id,
name: row.name,
type: row.type,
filePath: row.file_path,
location: JSON.parse(row.location),
metadata: row.metadata ? JSON.parse(row.metadata) : {},
hash: row.hash,
createdAt: row.created_at,
updatedAt: row.updated_at,
complexityScore: row.complexity_score,
language: row.language,
sizeBytes: row.size_bytes
};
}
/**
* Convert database row to Relationship (enhanced for v2)
*/
rowToRelationship(row) {
return {
id: row.id,
fromId: row.from_id,
toId: row.to_id,
type: row.type,
metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
weight: row.weight,
createdAt: row.created_at
};
}
/**
* Clear all data (for testing)
*/
async clear() {
const transaction = this.db.transaction(() => {
this.db.exec("DELETE FROM relationships");
this.db.exec("DELETE FROM entities");
this.db.exec("DELETE FROM files");
this.db.exec("DELETE FROM query_cache");
});
transaction();
}
};
}
});
function getSQLiteManager(config2) {
if (!instance) {
instance = new SQLiteManager(config2);
}
return instance;
}
var DEFAULT_DB_PATH, WAL_AUTOCHECKPOINT, CACHE_SIZE_KB, MMAP_SIZE, PAGE_SIZE, BUSY_TIMEOUT, SQLiteManager, instance;
var init_sqlite_manager = __esm({
"src/storage/sqlite-manager.ts"() {
DEFAULT_DB_PATH = join(homedir(), ".code-graph-rag", "codegraph.db");
WAL_AUTOCHECKPOINT = 1e3;
CACHE_SIZE_KB = 64e3;
MMAP_SIZE = 3e10;
PAGE_SIZE = 4096;
BUSY_TIMEOUT = 5e3;
SQLiteManager = class {
db = null;
config;
queryCount = 0;
totalQueryTime = 0;
constructor(config2 = {}) {
this.config = {
path: config2.path || DEFAULT_DB_PATH,
readonly: config2.readonly || false,
memory: config2.memory || false,
verbose: config2.verbose || false,
timeout: config2.timeout || BUSY_TIMEOUT
};
}
/**
* Initialize database connection with optimized settings
*/
initialize() {
if (this.db) {
console.warn("[SQLiteManager] Database already initialized");
return;
}
if (!this.config.memory && !this.config.readonly) {
const dir = dirname(this.config.path);
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true });
}
}
const dbPath = this.config.memory ? ":memory:" : this.config.path;
this.db = new Database(dbPath, {
readonly: this.config.readonly,
verbose: this.config.verbose ? console.log : void 0,
timeout: this.config.timeout
});
this.applyOptimizations();
console.log(`[SQLiteManager] Database initialized at ${dbPath}`);
}
/**
* Apply optimized PRAGMA settings for commodity hardware
*/
applyOptimizations() {
if (!this.db) throw new Error("Database not initialized");
this.db.pragma("journal_mode = WAL");
this.db.pragma(`cache_size = -${CACHE_SIZE_KB}`);
this.db.pragma("synchronous = NORMAL");
this.db.pragma("temp_store = MEMORY");
this.db.pragma(`mmap_size = ${MMAP_SIZE}`);
this.db.pragma(`page_size = ${PAGE_SIZE}`);
this.db.pragma(`wal_autocheckpoint = ${WAL_AUTOCHECKPOINT}`);
this.db.pragma("foreign_keys = ON");
if (!this.config.memory && !this.config.readonly) {
try {
this.db.exec("ANALYZE");
} catch (error) {
console.debug("[SQLiteManager] ANALYZE skipped (likely empty database)");
}
}
}
/**
* Get database connection
*/
getConnection() {
if (!this.db) {
throw new Error("Database not initialized. Call initialize() first.");
}
return this.db;
}
/**
* Prepare a statement with timing
*/
prepare(sql) {
const db = this.getConnection();
const statement = db.prepare(sql);
const originalRun = statement.run.bind(statement);
const originalGet = statement.get.bind(statement);
const originalAll = statement.all.bind(statement);
statement.run = (...args2) => {
const start = Date.now();
const result = originalRun(...args2);
this.recordQueryTime(Date.now() - start);
return result;
};
statement.get = (...args2) => {
const start = Date.now();
const result = originalGet(...args2);
this.recordQueryTime(Date.now() - start);
return result;
};
statement.all = (...args2) => {
const start = Date.now();
const result = originalAll(...args2);
this.recordQueryTime(Date.now() - start);
return result;
};
return statement;
}
/**
* Execute a transaction
*/
transaction(fn) {
const db = this.getConnection();
const start = Date.now();
const transaction = db.transaction(fn);
const result = transaction();
this.recordQueryTime(Date.now() - start);
return result;
}
/**
* Run VACUUM to optimize database
*/
vacuum() {
const db = this.getConnection();
console.log("[SQLiteManager] Running VACUUM...");
const start = Date.now();
db.exec("VACUUM");
const duration = Date.now() - start;
console.log(`[SQLiteManager] VACUUM completed in ${duration}ms`);
}
/**
* Run ANALYZE to update query optimizer statistics
*/
analyze() {
const db = this.getConnection();
console.log("[SQLiteManager] Running ANALYZE...");
const start = Date.now();
db.exec("ANALYZE");
const duration = Date.now() - start;
console.log(`[SQLiteManager] ANALYZE completed in ${duration}ms`);
}
/**
* Checkpoint WAL file
*/
checkpoint() {
const db = this.getConnection();
const result = db.pragma("wal_checkpoint(TRUNCATE)");
console.log("[SQLiteManager] WAL checkpoint completed", result);
}
/**
* Get database information
*/
getInfo() {
const db = this.getConnection();
return {
version: db.pragma("user_version", { simple: true }),
pageSize: db.pragma("page_size", { simple: true }),
pageCount: db.pragma("page_count", { simple: true }),
sizeBytes: db.pragma("page_count", { simple: true }) * db.pragma("page_size", { simple: true }),
journalMode: db.pragma("journal_mode", { simple: true }),
cacheSize: Math.abs(db.pragma("cache_size", { simple: true })),
walCheckpoint: db.pragma("wal_autocheckpoint", { simple: true })
};
}
/**
* Get storage metrics
*/
async getMetrics() {
const info = this.getInfo();
const db = this.getConnection();
const entityCount = db.prepare("SELECT COUNT(*) as count FROM entities").get();
const relationshipCount = db.prepare("SELECT COUNT(*) as count FROM relationships").get();
const fileCount = db.prepare("SELECT COUNT(*) as count FROM files").get();
const indexInfo = db.prepare(`
SELECT SUM(pgsize) as size
FROM dbstat
WHERE name LIKE 'idx_%'
`).get();
return {
totalEntities: entityCount?.count || 0,
totalRelationships: relationshipCount?.count || 0,
totalFiles: fileCount?.count || 0,
databaseSizeMB: info.sizeBytes / (1024 * 1024),
indexSizeMB: (indexInfo?.size || 0) / (1024 * 1024),
averageQueryTimeMs: this.queryCount > 0 ? this.totalQueryTime / this.queryCount : 0
};
}
/**
* Close database connection
*/
close() {
if (this.db) {
this.db.close();
this.db = null;
console.log("[SQLiteManager] Database connection closed");
}
}
/**
* Check if database is open
*/
isOpen() {
return this.db !== null && this.db.open;
}
/**
* Record query time for metrics
*/
recordQueryTime(timeMs) {
this.queryCount++;
this.totalQueryTime += timeMs;
}
/**
* Reset query metrics
*/
resetMetrics() {
this.queryCount = 0;
this.totalQueryTime = 0;
}
/**
* Enable or disable verbose logging
*/
setVerbose(verbose) {
if (this.db) {
this.db.function("log", (msg) => console.log(`[SQLite] ${msg}`));
if (verbose) {
this.db.pragma("vdbe_trace = ON");
} else {
this.db.pragma("vdbe_trace = OFF");
}
}
}
};
instance = null;
}
});
function runMigrations(sqliteManager) {
const migration = new SchemaMigration(sqliteManager);
migration.migrate();
}
var MIGRATIONS_TABLE, CURRENT_VERSION, migrations, SchemaMigration;
var init_schema_migrations = __esm({
"src/storage/schema-migrations.ts"() {
MIGRATIONS_TABLE = "migrations";
CURRENT_VERSION = 2;
migrations = [
{
version: 1,
description: "Initial schema with entities, relationships, and files",
up: `
-- Entities table for storing code elements
CREATE TABLE IF NOT EXISTS entities (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
file_path TEXT NOT NULL,
location TEXT NOT NULL, -- JSON: {start: {line, column, index}, end: {...}}
metadata TEXT, -- JSON: {modifiers, returnType, parameters, etc.}
hash TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
-- Indexes for efficient entity queries
CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(type);
CREATE INDEX IF NOT EXISTS idx_entities_file ON entities(file_path);
CREATE INDEX IF NOT EXISTS idx_entities_hash ON entities(hash);
CREATE INDEX IF NOT EXISTS idx_entities_type_name ON entities(type, name);
-- Relationships table for entity connections
CREATE TABLE IF NOT EXISTS relationships (
id TEXT PRIMARY KEY,
from_id TEXT NOT NULL,
to_id TEXT NOT NULL,
type TEXT NOT NULL,
metadata TEXT, -- JSON: {line, column, context}
FOREIGN KEY (from_id) REFERENCES entities(id) ON DELETE CASCADE,
FOREIGN KEY (to_id) REFERENCES entities(id) ON DELETE CASCADE
);
-- Indexes for efficient relationship queries
CREATE INDEX IF NOT EXISTS idx_rel_from ON relationships(from_id);
CREATE INDEX IF NOT EXISTS idx_rel_to ON relationships(to_id);
CREATE INDEX IF NOT EXISTS idx_rel_type ON relationships(type);
CREATE INDEX IF NOT EXISTS idx_rel_from_type ON relationships(from_id, type);
CREATE INDEX IF NOT EXISTS idx_rel_to_type ON relationships(to_id, type);
-- Files table for tracking indexed files
CREATE TABLE IF NOT EXISTS files (
path TEXT PRIMARY KEY,
hash TEXT NOT NULL,
last_indexed INTEGER NOT NULL,
entity_count INTEGER NOT NULL DEFAULT 0
);
-- Index for finding outdated files
CREATE INDEX IF NOT EXISTS idx_files_indexed ON files(last_indexed);
-- Query cache table for performance
CREATE TABLE IF NOT EXISTS query_cache (
cache_key TEXT PRIMARY KEY,
result TEXT NOT NULL, -- JSON serialized result
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
hit_count INTEGER DEFAULT 0
);
-- Index for cache expiration
CREATE INDEX IF NOT EXISTS idx_cache_expires ON query_cache(expires_at);
-- Full-text search virtual table for entity names and metadata
CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(
id UNINDEXED,
name,
type,
file_path,
metadata,
content=entities,
content_rowid=rowid
);
-- Triggers to keep FTS index in sync
CREATE TRIGGER IF NOT EXISTS entities_fts_insert
AFTER INSERT ON entities
BEGIN
INSERT INTO entities_fts(rowid, id, name, type, file_path, metadata)
VALUES (new.rowid, new.id, new.name, new.type, new.file_path, new.metadata);
END;
CREATE TRIGGER IF NOT EXISTS entities_fts_delete
AFTER DELETE ON entities
BEGIN
DELETE FROM entities_fts WHERE rowid = old.rowid;
END;
CREATE TRIGGER IF NOT EXISTS entities_fts_update
AFTER UPDATE ON entities
BEGIN
DELETE FROM entities_fts WHERE rowid = old.rowid;
INSERT INTO entities_fts(rowid, id, name, type, file_path, metadata)
VALUES (new.rowid, new.id, new.name, new.type, new.file_path, new.metadata);
END;
`,
down: `
DROP TRIGGER IF EXISTS entities_fts_update;
DROP TRIGGER IF EXISTS entities_fts_delete;
DROP TRIGGER IF EXISTS entities_fts_insert;
DROP TABLE IF EXISTS entities_fts;
DROP TABLE IF EXISTS query_cache;
DROP TABLE IF EXISTS files;
DROP TABLE IF EXISTS relationships;
DROP TABLE IF EXISTS entities;
`
},
{
version: 2,
description: "Enhanced schema with optimized indexing and vector integration",
up: `
-- Enhanced entities table with performance optimizations
ALTER TABLE entities ADD COLUMN complexity_score INTEGER DEFAULT 0;
ALTER TABLE entities ADD COLUMN language TEXT;
ALTER TABLE entities ADD COLUMN size_bytes INTEGER DEFAULT 0;
-- Additional performance indexes
CREATE INDEX IF NOT EXISTS idx_entities_complexity ON entities(complexity_score);
CREATE INDEX IF NOT EXISTS idx_entities_language ON entities(language);
CREATE INDEX IF NOT EXISTS idx_entities_size ON entities(size_bytes);
-- Compound indexes for common query patterns
CREATE INDEX IF NOT EXISTS idx_entities_file_type ON entities(file_path, type);
CREATE INDEX IF NOT EXISTS idx_entities_lang_type ON entities(language, type);
CREATE INDEX IF NOT EXISTS idx_entities_updated_type ON entities(updated_at, type);
-- Enhanced relationships with metadata indexing
ALTER TABLE relationships ADD COLUMN weight REAL DEFAULT 1.0;
ALTER TABLE relationships ADD COLUMN created_at INTEGER DEFAULT 0;
-- Performance indexes for relationships
CREATE INDEX IF NOT EXISTS idx_rel_weight ON relationships(weight);
CREATE INDEX IF NOT EXISTS idx_rel_created ON relationships(created_at);
CREATE INDEX IF NOT EXISTS idx_rel_from_to_type ON relationships(from_id, to_id, type);
-- Vector embeddings table (separate from metadata)
CREATE TABLE IF NOT EXISTS embeddings (
id TEXT PRIMARY KEY,
entity_id TEXT NOT NULL,
content TEXT NOT NULL,
vector_data BLOB,
model_name TEXT NOT NULL DEFAULT 'default',
created_at INTEGER NOT NULL,
FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE CASCADE
);
-- Indexes for embeddings
CREATE INDEX IF NOT EXISTS idx_embeddings_entity ON embeddings(entity_id);
CREATE INDEX IF NOT EXISTS idx_embeddings_model ON embeddings(model_name);
CREATE INDEX IF NOT EXISTS idx_embeddings_created ON embeddings(created_at);
-- Performance monitoring table
CREATE TABLE IF NOT EXISTS performance_metrics (
id TEXT PRIMARY KEY,
operation TEXT NOT NULL,
duration_ms INTEGER NOT NULL,
entity_count INTEGER DEFAULT 0,
memory_usage INTEGER DEFAULT 0,
created_at INTEGER NOT NULL
);
-- Index for performance metrics
CREATE INDEX IF NOT EXISTS idx_perf_operation ON performance_metrics(operation);
CREATE INDEX IF NOT EXISTS idx_perf_created ON performance_metrics(created_at);
-- Enhanced query cache with statistics
ALTER TABLE query_cache ADD COLUMN miss_count INTEGER DEFAULT 0;
ALTER TABLE query_cache ADD COLUMN last_accessed INTEGER;
-- Index for cache performance analysis
CREATE INDEX IF NOT EXISTS idx_cache_accessed ON query_cache(last_accessed);
CREATE INDEX IF NOT EXISTS idx_cache_hits ON query_cache(hit_count);
-- Update existing data with