@adarsh6938/mcp-knowledge-graph-semantic
Version:
Private MCP Server for semantic knowledge graph with persistent memory
1,056 lines • 82.8 kB
JavaScript
#!/usr/bin/env node
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
import { promises as fs } from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import minimist from 'minimist';
import { isAbsolute } from 'path';
import { pipeline } from '@xenova/transformers';
// Parse args and handle paths safely
const argv = minimist(process.argv.slice(2));
// Check for memory path in command line args or environment variable
let memoryPath = argv['memory-path'] || process.env.MEMORY_FILE_PATH;
// If a custom path is provided, ensure it's absolute
if (memoryPath && !isAbsolute(memoryPath)) {
memoryPath = path.resolve(process.cwd(), memoryPath);
}
// Define the path to the JSONL file
const __dirname = path.dirname(fileURLToPath(import.meta.url));
// Use the custom path or default to the installation directory
const MEMORY_FILE_PATH = memoryPath || path.join(__dirname, 'memory.jsonl');
// Create embeddings file path based on memory file name
const getEmbeddingsFilePath = (memoryFilePath) => {
const parsedPath = path.parse(memoryFilePath);
return path.join(parsedPath.dir, `${parsedPath.name}_embeddings.json`);
};
const EMBEDDINGS_FILE_PATH = getEmbeddingsFilePath(MEMORY_FILE_PATH);
// Create session summaries file path based on memory file name
const getSessionSummariesFilePath = (memoryFilePath) => {
const parsedPath = path.parse(memoryFilePath);
return path.join(parsedPath.dir, `${parsedPath.name}_session_summaries.json`);
};
const SESSION_SUMMARIES_FILE_PATH = getSessionSummariesFilePath(MEMORY_FILE_PATH);
// Configuration constants - Can be customized per use case
const DEFAULT_MAX_OBSERVATIONS_PER_ENTITY = 20;
const DEFAULT_WARNING_THRESHOLD = 12;
const DEFAULT_OPTIMAL_OBSERVATIONS_COUNT = 8;
// Default configuration
const DEFAULT_CONFIG = {
maxObservationsPerEntity: DEFAULT_MAX_OBSERVATIONS_PER_ENTITY,
warningThreshold: DEFAULT_WARNING_THRESHOLD,
optimalObservationsCount: DEFAULT_OPTIMAL_OBSERVATIONS_COUNT,
enableSmartSuggestions: true,
enableAutoSplit: true
};
// Smart categorization rules - Generic patterns for universal applicability
const OBSERVATION_CATEGORIES = [
{
name: "activities_and_actions",
patterns: [
/\b(working on|developing|building|creating|implementing|designing)\b/i,
/\b(completed|finished|started|began|initiated)\b/i,
/\b(managing|leading|coordinating|organizing)\b/i,
/\b(learning|studying|researching|investigating)\b/i
],
suggestedEntityType: "activity",
suggestedRelationType: "performs_activity"
},
{
name: "tools_and_technologies",
patterns: [
/\b(uses|utilizing|working with|experienced with)\b.*\b(software|tool|platform|system|application)\b/i,
/\b(framework|library|database|programming|language)\b/i,
/\b(proficient in|skilled in|expertise in|knowledge of)\b/i,
/\b(version|v\d+|\d+\.\d+)\b/i
],
suggestedEntityType: "tool",
suggestedRelationType: "uses_tool"
},
{
name: "problem_solving",
patterns: [
/\b(problem|issue|bug|error|challenge|difficulty)\b/i,
/\b(solved|resolved|fixed|addressed|troubleshooting)\b/i,
/\b(solution|workaround|approach|method|strategy)\b/i,
/\b(debugging|investigating|analyzing|diagnosing)\b/i
],
suggestedEntityType: "problem",
suggestedRelationType: "encountered_problem"
},
{
name: "knowledge_and_learning",
patterns: [
/\b(learned|discovered|found out|realized|understood)\b/i,
/\b(knowledge|understanding|insight|concept|principle)\b/i,
/\b(studied|researched|explored|investigated)\b/i,
/\b(documentation|tutorial|guide|course|training)\b/i
],
suggestedEntityType: "knowledge_area",
suggestedRelationType: "has_knowledge_of"
},
{
name: "projects_and_goals",
patterns: [
/\b(project|initiative|goal|objective|target)\b/i,
/\b(milestone|deadline|timeline|schedule|plan)\b/i,
/\b(progress|status|update|achievement|accomplishment)\b/i,
/\b(collaboration|teamwork|partnership)\b/i
],
suggestedEntityType: "project",
suggestedRelationType: "works_on"
},
{
name: "relationships_and_interactions",
patterns: [
/\b(met with|discussed with|collaborated with|worked with)\b/i,
/\b(team|colleague|partner|client|customer|user)\b/i,
/\b(meeting|discussion|conversation|communication)\b/i,
/\b(feedback|review|input|suggestion|recommendation)\b/i
],
suggestedEntityType: "interaction",
suggestedRelationType: "had_interaction"
},
{
name: "processes_and_workflows",
patterns: [
/\b(process|procedure|workflow|methodology|approach)\b/i,
/\b(step|phase|stage|sequence|order)\b/i,
/\b(automation|optimization|improvement|efficiency)\b/i,
/\b(best practice|standard|guideline|protocol)\b/i
],
suggestedEntityType: "process",
suggestedRelationType: "follows_process"
}
];
// Semantic Search Manager
class SemanticSearchManager {
constructor() {
this.embedder = null;
this.modelName = 'Xenova/all-MiniLM-L6-v2';
}
async initializeEmbedder() {
if (!this.embedder) {
this.embedder = await pipeline('feature-extraction', this.modelName);
}
return this.embedder;
}
async generateEmbedding(text) {
const embedder = await this.initializeEmbedder();
const output = await embedder(text, { pooling: 'mean', normalize: true });
return Array.from(output.data);
}
cosineSimilarity(a, b) {
const dotProduct = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
const magnitudeA = Math.sqrt(a.reduce((sum, ai) => sum + ai * ai, 0));
const magnitudeB = Math.sqrt(b.reduce((sum, bi) => sum + bi * bi, 0));
return dotProduct / (magnitudeA * magnitudeB);
}
async loadSemanticIndex() {
try {
const data = await fs.readFile(EMBEDDINGS_FILE_PATH, 'utf-8');
return JSON.parse(data);
}
catch (error) {
return {
model: this.modelName,
embeddings: [],
version: 1
};
}
}
async saveSemanticIndex(index) {
await fs.writeFile(EMBEDDINGS_FILE_PATH, JSON.stringify(index, null, 2));
}
async buildSemanticIndex(entities) {
const index = await this.loadSemanticIndex();
const newEmbeddings = [];
for (const entity of entities) {
// Create embedding for entity name + type
const entityText = `${entity.name} ${entity.entityType}`;
const existingEntityEmbedding = index.embeddings.find(e => e.entityName === entity.name && e.type === 'entity');
if (!existingEntityEmbedding) {
const embedding = await this.generateEmbedding(entityText);
newEmbeddings.push({
text: entityText,
embedding,
entityName: entity.name,
type: 'entity',
lastUpdated: new Date().toISOString()
});
}
// Create embeddings for observations
for (const observation of entity.observations) {
const observationText = ObservationUtils.toString(observation);
const existingObsEmbedding = index.embeddings.find(e => e.entityName === entity.name && e.type === 'observation' && e.text === observationText);
if (!existingObsEmbedding) {
const embedding = await this.generateEmbedding(observationText);
newEmbeddings.push({
text: observationText,
embedding,
entityName: entity.name,
type: 'observation',
lastUpdated: new Date().toISOString()
});
}
}
}
if (newEmbeddings.length > 0) {
index.embeddings.push(...newEmbeddings);
index.version += 1;
await this.saveSemanticIndex(index);
}
}
async semanticSearch(query, entities, threshold = 0.3, maxResults = 10) {
const index = await this.loadSemanticIndex();
if (index.embeddings.length === 0) {
await this.buildSemanticIndex(entities);
return this.semanticSearch(query, entities, threshold, maxResults);
}
const queryEmbedding = await this.generateEmbedding(query);
const results = [];
const now = new Date().getTime();
for (const embeddingData of index.embeddings) {
const similarity = this.cosineSimilarity(queryEmbedding, embeddingData.embedding);
if (similarity >= threshold) {
const entity = entities.find(e => e.name === embeddingData.entityName);
if (entity) {
// Calculate recency score (1.0 = very recent, 0.0 = very old)
const entityDate = new Date(entity.createdAt).getTime();
const daysSinceCreation = (now - entityDate) / (1000 * 60 * 60 * 24);
const recencyScore = Math.max(0, 1.0 - (daysSinceCreation / 365)); // Decay over 1 year
// Combined score: 70% similarity + 30% recency
const combinedScore = (similarity * 0.7) + (recencyScore * 0.3);
results.push({
entity,
similarity: combinedScore, // Store combined score for sorting
matchedText: embeddingData.text,
matchType: embeddingData.type
});
}
}
}
return results
.sort((a, b) => b.similarity - a.similarity) // Now sorts by combined score
.slice(0, maxResults);
}
async rebuildIndex(entities) {
// Delete existing index and rebuild
try {
await fs.unlink(EMBEDDINGS_FILE_PATH);
}
catch (error) {
// File doesn't exist, that's fine
}
await this.buildSemanticIndex(entities);
}
}
// The KnowledgeGraphManager class contains all operations to interact with the knowledge graph
class KnowledgeGraphManager {
constructor() {
this.semanticSearch = new SemanticSearchManager();
this.config = DEFAULT_CONFIG;
this.activeCategories = OBSERVATION_CATEGORIES;
}
// Allow users to configure entity management behavior
configureEntityManagement(config) {
this.config = { ...DEFAULT_CONFIG, ...config };
// Use custom categories if provided, otherwise use defaults
if (config.customCategories && config.customCategories.length > 0) {
this.activeCategories = config.customCategories;
}
else {
this.activeCategories = OBSERVATION_CATEGORIES;
}
}
// Get current configuration
getEntityManagementConfig() {
return { ...this.config };
}
async loadGraph() {
try {
const data = await fs.readFile(MEMORY_FILE_PATH, "utf-8");
const lines = data.split("\n").filter(line => line.trim() !== "");
return lines.reduce((graph, line) => {
const item = JSON.parse(line);
if (item.type === "entity")
graph.entities.push(item);
if (item.type === "relation")
graph.relations.push(item);
return graph;
}, { entities: [], relations: [] });
}
catch (error) {
if (error instanceof Error && 'code' in error && error.code === "ENOENT") {
return { entities: [], relations: [] };
}
throw error;
}
}
async saveGraph(graph) {
const lines = [
...graph.entities.map(e => JSON.stringify({ type: "entity", ...e })),
...graph.relations.map(r => JSON.stringify({ type: "relation", ...r })),
];
await fs.writeFile(MEMORY_FILE_PATH, lines.join("\n"));
}
async createEntities(entities) {
const graph = await this.loadGraph();
const newEntities = entities.filter(e => !graph.entities.some(existingEntity => existingEntity.name === e.name))
.map(e => ({
...e,
createdAt: new Date().toISOString(),
version: e.version || 1
}));
graph.entities.push(...newEntities);
await this.saveGraph(graph);
// Update semantic index for new entities
if (newEntities.length > 0) {
await this.semanticSearch.buildSemanticIndex(newEntities);
}
return newEntities;
}
async createRelations(relations) {
const graph = await this.loadGraph();
const newRelations = relations.filter(r => !graph.relations.some(existingRelation => existingRelation.from === r.from &&
existingRelation.to === r.to &&
existingRelation.relationType === r.relationType)).map(r => ({
...r,
createdAt: new Date().toISOString(),
version: r.version || 1
}));
graph.relations.push(...newRelations);
await this.saveGraph(graph);
return newRelations;
}
async addObservations(observations) {
const graph = await this.loadGraph();
const results = [];
for (const o of observations) {
const entity = graph.entities.find(e => e.name === o.entityName);
if (!entity) {
throw new Error(`Entity with name ${o.entityName} not found`);
}
const warnings = [];
const suggestions = [];
// Check current observation count and warn if approaching limits
const currentCount = entity.observations.length;
const newCount = currentCount + o.contents.length;
// Check if auto-split should be performed
if (newCount > (this.config.maxObservationsPerEntity || DEFAULT_MAX_OBSERVATIONS_PER_ENTITY)) {
if (this.config.enableAutoSplit !== false) {
// Perform automatic entity splitting
console.error(`🤖 Auto-splitting entity "${entity.name}" (${currentCount} + ${o.contents.length} = ${newCount} observations exceed limit)`);
// Combine existing and new observations for categorization (convert all to strings)
const existingStrings = entity.observations.map(obs => ObservationUtils.toString(obs));
const allObservationStrings = [...existingStrings, ...o.contents];
const categories = {};
const uncategorized = [];
// Categorize all observations
for (const obsString of allObservationStrings) {
const category = this.categorizeObservation(obsString);
if (category) {
if (!categories[category.name]) {
categories[category.name] = [];
}
categories[category.name].push(obsString);
}
else {
uncategorized.push(obsString);
}
}
// If we have multiple categories, split the entity
const categoryKeys = Object.keys(categories);
if (categoryKeys.length > 1) {
// Clear the original entity observations
entity.observations = [];
// Create new entities for each category
const newEntityNames = [];
for (const categoryName of categoryKeys) {
const categoryObservations = categories[categoryName];
if (categoryObservations.length > 0) {
const newEntityName = `${entity.name}_${categoryName}`;
const categoryInfo = this.activeCategories.find(c => c.name === categoryName);
// Create new entity with enhanced observations
const enhancedObservations = categoryObservations.map(content => ObservationUtils.createEnhanced(content));
const newEntity = {
name: newEntityName,
entityType: categoryInfo?.suggestedEntityType || `${entity.entityType}_${categoryName}`,
observations: enhancedObservations,
createdAt: new Date().toISOString(),
version: 1,
sessionId: SessionManager.getCurrentSessionId()
};
graph.entities.push(newEntity);
newEntityNames.push(newEntityName);
// Create relation from original to new entity
const newRelation = {
from: entity.name,
to: newEntityName,
relationType: categoryInfo?.suggestedRelationType || `has_${categoryName}`,
createdAt: new Date().toISOString(),
version: 1,
sessionId: SessionManager.getCurrentSessionId()
};
graph.relations.push(newRelation);
}
}
// Handle uncategorized observations (keep in original entity)
if (uncategorized.length > 0) {
entity.observations = uncategorized.map(content => ObservationUtils.createEnhanced(content));
}
// Filter out duplicates from new observations (already added during split)
const newObservationStrings = o.contents.filter(content => !existingStrings.some(existing => existing === content));
const enhancedNewObs = newObservationStrings.map(content => ObservationUtils.createEnhanced(content));
entity.observations.push(...enhancedNewObs);
results.push({
entityName: o.entityName,
addedObservations: o.contents,
warnings: [`Entity automatically split into ${newEntityNames.length} specialized entities`],
autoSplitPerformed: true,
newEntities: newEntityNames
});
}
else {
// If only one category or no categorization, just raise the limit and warn
const existingContents = entity.observations.map(obs => ObservationUtils.toString(obs));
const newObservationStrings = o.contents.filter(content => !existingContents.includes(content));
const enhancedNewObs = newObservationStrings.map(content => ObservationUtils.createEnhanced(content));
entity.observations.push(...enhancedNewObs);
warnings.push(`Entity "${entity.name}" now has ${entity.observations.length} observations. Consider manual organization.`);
results.push({
entityName: o.entityName,
addedObservations: newObservationStrings,
warnings
});
}
}
else {
// Auto-split disabled, throw error as before
throw new Error(`⚠️ Entity "${entity.name}" would exceed ${this.config.maxObservationsPerEntity || DEFAULT_MAX_OBSERVATIONS_PER_ENTITY} observations (currently has ${currentCount}, trying to add ${o.contents.length}). Consider creating separate entities for different domains.`);
}
}
else {
// Normal case - entity is within limits
if (newCount > (this.config.warningThreshold || DEFAULT_WARNING_THRESHOLD)) {
warnings.push(`Entity "${entity.name}" will have ${newCount} observations. Consider splitting into separate entities.`);
}
// Analyze each observation for smart categorization
if (this.config.enableSmartSuggestions !== false) {
for (const content of o.contents) {
const category = this.categorizeObservation(content);
if (category && entity.entityType === "Person") {
// Suggest creating separate entities for technical content added to person entities
const suggestedEntityName = `${entity.name}_${category.name}`;
suggestions.push({
originalObservation: content,
category,
suggestedEntityName,
suggestedEntityType: category.suggestedEntityType,
suggestedRelationType: category.suggestedRelationType
});
}
}
}
const existingContents = entity.observations.map(obs => ObservationUtils.toString(obs));
const newObservationStrings = o.contents.filter(content => !existingContents.includes(content));
const enhancedNewObs = newObservationStrings.map(content => ObservationUtils.createEnhanced(content));
entity.observations.push(...enhancedNewObs);
const result = { entityName: o.entityName, addedObservations: newObservationStrings };
if (warnings.length > 0)
result.warnings = warnings;
if (suggestions.length > 0)
result.suggestions = suggestions;
results.push(result);
}
}
await this.saveGraph(graph);
// Update semantic index for all affected entities
const updatedEntities = observations.map(o => graph.entities.find(e => e.name === o.entityName)).filter(Boolean);
// Also include any new entities created during auto-split
const newEntities = graph.entities.filter(e => results.some(r => r.newEntities?.includes(e.name)));
await this.semanticSearch.buildSemanticIndex([...updatedEntities, ...newEntities]);
return results;
}
categorizeObservation(observation) {
for (const category of this.activeCategories) {
if (category.patterns.some(pattern => pattern.test(observation))) {
return category;
}
}
return null;
}
async deleteEntities(entityNames) {
const graph = await this.loadGraph();
graph.entities = graph.entities.filter(e => !entityNames.includes(e.name));
graph.relations = graph.relations.filter(r => !entityNames.includes(r.from) && !entityNames.includes(r.to));
await this.saveGraph(graph);
}
async deleteObservations(deletions) {
const graph = await this.loadGraph();
deletions.forEach(d => {
const entity = graph.entities.find(e => e.name === d.entityName);
if (entity) {
entity.observations = entity.observations.filter(o => {
const obsText = ObservationUtils.toString(o);
return !d.observations.includes(obsText);
});
}
});
await this.saveGraph(graph);
}
async deleteRelations(relations) {
const graph = await this.loadGraph();
graph.relations = graph.relations.filter(r => !relations.some(delRelation => r.from === delRelation.from &&
r.to === delRelation.to &&
r.relationType === delRelation.relationType));
await this.saveGraph(graph);
}
async readGraph() {
// Return a summary instead of full graph to avoid size issues
const graph = await this.loadGraph();
// Limit to first 5 entities and their relations to prevent response size issues
const limitedEntities = graph.entities.slice(0, 5);
const limitedEntityNames = new Set(limitedEntities.map(e => e.name));
const limitedRelations = graph.relations.filter(r => limitedEntityNames.has(r.from) && limitedEntityNames.has(r.to)).slice(0, 10);
return {
entities: limitedEntities,
relations: limitedRelations
};
}
async readGraphPaginated(page = 0, pageSize = 5) {
const graph = await this.loadGraph();
const startIndex = page * pageSize;
const endIndex = startIndex + pageSize;
const paginatedEntities = graph.entities.slice(startIndex, endIndex);
const paginatedEntityNames = new Set(paginatedEntities.map(e => e.name));
// Get relations involving these entities
const relevantRelations = graph.relations.filter(r => paginatedEntityNames.has(r.from) || paginatedEntityNames.has(r.to));
return {
entities: paginatedEntities,
relations: relevantRelations,
pagination: {
page,
pageSize,
totalEntities: graph.entities.length,
totalRelations: graph.relations.length,
hasMore: endIndex < graph.entities.length
}
};
}
// Very basic search function
async searchNodes(query) {
const graph = await this.loadGraph();
const lowerQuery = query.toLowerCase();
const matchingEntities = graph.entities.filter(entity => {
const entityMatch = entity.name.toLowerCase().includes(lowerQuery) ||
entity.entityType.toLowerCase().includes(lowerQuery);
const observationMatch = entity.observations.some(observation => {
const obsText = ObservationUtils.toString(observation);
return obsText.toLowerCase().includes(lowerQuery);
});
return entityMatch || observationMatch;
});
const matchingEntityNames = new Set(matchingEntities.map(e => e.name));
const matchingRelations = graph.relations.filter(relation => matchingEntityNames.has(relation.from) && matchingEntityNames.has(relation.to));
return {
entities: matchingEntities,
relations: matchingRelations
};
}
// Semantic search function
async semanticSearchNodes(query, threshold = 0.3, maxResults = 10) {
const graph = await this.loadGraph();
const results = await this.semanticSearch.semanticSearch(query, graph.entities, threshold, maxResults);
// Create a filtered graph with the semantic search results
const filteredEntities = results.map(r => r.entity);
const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to));
return {
results,
graph: {
entities: filteredEntities,
relations: filteredRelations,
}
};
}
// Hybrid search combining keyword and semantic search
async hybridSearch(query, semanticWeight = 0.7, threshold = 0.3, maxResults = 10) {
const graph = await this.loadGraph();
// Get semantic search results
const semanticResults = await this.semanticSearch.semanticSearch(query, graph.entities, threshold, maxResults * 2);
// Get keyword search results
const keywordGraph = await this.searchNodes(query);
// Combine and score results
const combinedResults = new Map();
// Add semantic results
semanticResults.forEach(result => {
combinedResults.set(result.entity.name, result);
});
// Boost entities that also match keyword search
keywordGraph.entities.forEach(entity => {
const existing = combinedResults.get(entity.name);
if (existing) {
// Boost similarity score for keyword matches
existing.similarity = existing.similarity * semanticWeight + (1 - semanticWeight);
existing.keywordMatch = true;
}
else {
// Add keyword-only matches with lower similarity
combinedResults.set(entity.name, {
entity,
similarity: 1 - semanticWeight,
matchedText: entity.name,
matchType: 'entity',
keywordMatch: true
});
}
});
const results = Array.from(combinedResults.values())
.sort((a, b) => b.similarity - a.similarity)
.slice(0, maxResults);
const filteredEntities = results.map(r => r.entity);
const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to));
return {
results,
graph: {
entities: filteredEntities,
relations: filteredRelations,
}
};
}
// Rebuild semantic index
async rebuildSemanticIndex() {
const graph = await this.loadGraph();
await this.semanticSearch.rebuildIndex(graph.entities);
}
async openNodes(names) {
const graph = await this.loadGraph();
const filteredEntities = graph.entities.filter(e => names.includes(e.name));
const filteredEntityNames = new Set(filteredEntities.map(e => e.name));
const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) || filteredEntityNames.has(r.to));
return {
entities: filteredEntities,
relations: filteredRelations,
};
}
async analyzeEntityHealth() {
const graph = await this.loadGraph();
const bloatedEntities = [];
const wellFormedEntities = [];
const warningThreshold = this.config.warningThreshold || DEFAULT_WARNING_THRESHOLD;
for (const entity of graph.entities) {
const observationCount = entity.observations.length;
if (observationCount > warningThreshold) {
// Analyze categories for this entity
const categories = {};
let uncategorizedCount = 0;
for (const observation of entity.observations) {
const obsText = ObservationUtils.toString(observation);
const category = this.categorizeObservation(obsText);
if (category) {
categories[category.name] = (categories[category.name] || 0) + 1;
}
else {
uncategorizedCount++;
}
}
if (uncategorizedCount > 0) {
categories['uncategorized'] = uncategorizedCount;
}
const suggestions = Object.keys(categories).length > 1
? [`Split into ${Object.keys(categories).length} entities based on categories: ${Object.keys(categories).join(', ')}`]
: ['Consider manual review and reorganization'];
bloatedEntities.push({
name: entity.name,
observationCount,
suggestions,
categories
});
}
else {
wellFormedEntities.push(entity.name);
}
}
const totalEntities = graph.entities.length;
const totalObservations = graph.entities.reduce((sum, entity) => sum + entity.observations.length, 0);
const averageObservationsPerEntity = totalEntities > 0 ? totalObservations / totalEntities : 0;
return {
bloatedEntities,
wellFormedEntities,
totalEntities,
averageObservationsPerEntity: Math.round(averageObservationsPerEntity * 100) / 100
};
}
async splitEntity(entityName, categories) {
const graph = await this.loadGraph();
const entity = graph.entities.find(e => e.name === entityName);
if (!entity) {
throw new Error(`Entity with name ${entityName} not found`);
}
const originalEntity = { ...entity };
const newEntities = [];
const newRelations = [];
// Validate that all observations to be moved exist in the entity
const allObservationsToMove = Object.values(categories).flat();
const entityObservationStrings = entity.observations.map(obs => ObservationUtils.toString(obs));
for (const obsToMove of allObservationsToMove) {
if (!entityObservationStrings.includes(obsToMove)) {
throw new Error(`Observation "${obsToMove}" not found in entity "${entityName}"`);
}
}
// Create new entities for each category
for (const [categoryName, observationsToMove] of Object.entries(categories)) {
if (observationsToMove.length === 0)
continue;
const newEntityName = `${entityName}_${categoryName}`;
// Find appropriate entity type and relation type based on category
const categoryInfo = this.activeCategories.find(c => c.name === categoryName);
const relationType = categoryInfo?.suggestedRelationType || `has_${categoryName}`;
// Create enhanced observations for the new entity
const enhancedObservations = observationsToMove.map(content => ObservationUtils.createEnhanced(content));
const newEntity = {
name: newEntityName,
entityType: categoryInfo?.suggestedEntityType || `${entity.entityType}_${categoryName}`,
observations: enhancedObservations,
createdAt: new Date().toISOString(),
version: 1,
sessionId: SessionManager.getCurrentSessionId()
};
newEntities.push(newEntity);
graph.entities.push(newEntity);
// Create relation from original entity to new entity
const newRelation = {
from: entityName,
to: newEntityName,
relationType: relationType,
createdAt: new Date().toISOString(),
version: 1,
sessionId: SessionManager.getCurrentSessionId()
};
newRelations.push(newRelation);
graph.relations.push(newRelation);
}
// Remove moved observations from original entity
entity.observations = entity.observations.filter(observation => {
const obsText = ObservationUtils.toString(observation);
return !allObservationsToMove.includes(obsText);
});
await this.saveGraph(graph);
// Update semantic index for all affected entities
await this.semanticSearch.buildSemanticIndex([entity, ...newEntities]);
const summary = `Split entity "${entityName}" into ${newEntities.length} new entities: ${newEntities.map(e => e.name).join(', ')}. Moved ${allObservationsToMove.length} observations total.`;
return {
originalEntity,
newEntities,
newRelations,
summary
};
}
async updateEntities(entities) {
const graph = await this.loadGraph();
const updatedEntities = entities.map(updateEntity => {
const existingEntity = graph.entities.find(e => e.name === updateEntity.name);
if (!existingEntity) {
throw new Error(`Entity with name ${updateEntity.name} not found`);
}
return {
...existingEntity,
...updateEntity,
version: existingEntity.version + 1,
createdAt: new Date().toISOString()
};
});
// Update entities in the graph
updatedEntities.forEach(updatedEntity => {
const index = graph.entities.findIndex(e => e.name === updatedEntity.name);
if (index !== -1) {
graph.entities[index] = updatedEntity;
}
});
await this.saveGraph(graph);
return updatedEntities;
}
async updateRelations(relations) {
const graph = await this.loadGraph();
const updatedRelations = relations.map(updateRelation => {
const existingRelation = graph.relations.find(r => r.from === updateRelation.from &&
r.to === updateRelation.to &&
r.relationType === updateRelation.relationType);
if (!existingRelation) {
throw new Error(`Relation not found`);
}
return {
...existingRelation,
...updateRelation,
version: existingRelation.version + 1,
createdAt: new Date().toISOString()
};
});
// Update relations in the graph
updatedRelations.forEach(updatedRelation => {
const index = graph.relations.findIndex(r => r.from === updatedRelation.from &&
r.to === updatedRelation.to &&
r.relationType === updatedRelation.relationType);
if (index !== -1) {
graph.relations[index] = updatedRelation;
}
});
await this.saveGraph(graph);
return updatedRelations;
}
// Tiered Context Retrieval System for Session Continuity
/**
* Get recent context - last 24 hours with priority on recency
*/
async getRecentContext(hoursBack = 24, maxResults = 20) {
const graph = await this.loadGraph();
const cutoffTime = new Date(Date.now() - hoursBack * 60 * 60 * 1000);
const recentObservations = [];
const recentEntityNames = new Set();
const activeSessions = new Set();
// Find all recent observations with timestamps
for (const entity of graph.entities) {
for (const obs of entity.observations) {
if (typeof obs === 'object' && obs.timestamp) {
const obsTime = new Date(obs.timestamp);
if (obsTime >= cutoffTime) {
const hoursAgo = (Date.now() - obsTime.getTime()) / (1000 * 60 * 60);
recentObservations.push({
entity: entity.name,
observation: obs,
hoursAgo
});
recentEntityNames.add(entity.name);
if (obs.sessionId) {
activeSessions.add(obs.sessionId);
}
}
}
}
// Also include entities created recently
const entityCreatedTime = new Date(entity.createdAt);
if (entityCreatedTime >= cutoffTime) {
recentEntityNames.add(entity.name);
if (entity.sessionId) {
activeSessions.add(entity.sessionId);
}
}
}
// Sort by recency and limit results
recentObservations.sort((a, b) => a.hoursAgo - b.hoursAgo);
const limitedObservations = recentObservations.slice(0, maxResults);
// Get entities and relations for recent activity
const recentEntities = graph.entities.filter(e => recentEntityNames.has(e.name));
const recentRelations = graph.relations.filter(r => recentEntityNames.has(r.from) || recentEntityNames.has(r.to));
return {
entities: recentEntities,
relations: recentRelations,
recentObservations: limitedObservations,
activeSessions: Array.from(activeSessions)
};
}
/**
* Get related work - semantically related content from past week
*/
async getRelatedWork(query, daysBack = 7, maxResults = 15) {
const graph = await this.loadGraph();
const cutoffTime = new Date(Date.now() - daysBack * 24 * 60 * 60 * 1000);
// Filter entities by time range
const timeRangedEntities = graph.entities.filter(entity => {
const entityTime = new Date(entity.createdAt);
return entityTime >= cutoffTime;
});
// Perform semantic search within time range
const searchResults = await this.semanticSearch.semanticSearch(query, timeRangedEntities, 0.3, maxResults);
// Group by session for context clustering
const sessionClusters = {};
for (const entity of timeRangedEntities) {
if (entity.sessionId) {
if (!sessionClusters[entity.sessionId]) {
sessionClusters[entity.sessionId] = [];
}
sessionClusters[entity.sessionId].push(entity);
}
}
// Get relevant relations
const entityNames = new Set(timeRangedEntities.map(e => e.name));
const relations = graph.relations.filter(r => entityNames.has(r.from) || entityNames.has(r.to));
return {
results: searchResults,
timeRangedEntities,
relations,
sessionClusters
};
}
/**
* Get historical overview - older context summaries and key entities
*/
async getHistoricalOverview(excludeDays = 7, maxResults = 10) {
const graph = await this.loadGraph();
const cutoffTime = new Date(Date.now() - excludeDays * 24 * 60 * 60 * 1000);
// Get historical entities (older than excludeDays)
const historicalEntities = graph.entities.filter(entity => {
const entityTime = new Date(entity.createdAt);
return entityTime < cutoffTime;
});
// Create entity summaries
const entitySummary = historicalEntities.map(entity => {
const activityTypes = new Set();
let observationCount = 0;
for (const obs of entity.observations) {
observationCount++;
if (typeof obs === 'object' && obs.activityType) {
activityTypes.add(obs.activityType);
}
}
return {
name: entity.name,
entityType: entity.entityType,
observationCount,
createdAt: entity.createdAt,
sessionId: entity.sessionId,
activityTypes: Array.from(activityTypes)
};
});
// Sort by observation count and recency for key entities
const keyEntities = historicalEntities
.sort((a, b) => {
const aScore = a.observations.length + (new Date(a.createdAt).getTime() / 1000000);
const bScore = b.observations.length + (new Date(b.createdAt).getTime() / 1000000);
return bScore - aScore;
})
.slice(0, maxResults);
// Historical relations
const historicalEntityNames = new Set(historicalEntities.map(e => e.name));
const relations = graph.relations.filter(r => historicalEntityNames.has(r.from) || historicalEntityNames.has(r.to));
// Session history analysis
const sessionMap = new Map();
for (const entity of historicalEntities) {
if (entity.sessionId) {
if (!sessionMap.has(entity.sessionId)) {
sessionMap.set(entity.sessionId, { entities: [], observations: [] });
}
const session = sessionMap.get(entity.sessionId);
session.entities.push(entity);
for (const obs of entity.observations) {
if (typeof obs === 'object') {
session.observations.push(obs);
}
}
}
}
const sessionHistory = Array.from(sessionMap.entries()).map(([sessionId, session]) => {
const timestamps = session.observations
.map(obs => new Date(obs.timestamp))
.filter(date => !isNaN(date.getTime()));
const activityTypes = new Set(session.observations
.filter(obs => obs.activityType)
.map(obs => obs.activityType));
return {
sessionId,
entityCount: session.entities.length,
observationCount: session.observations.length,
timeRange: timestamps.length > 0 ? {
start: new Date(Math.min(...timestamps.map(d => d.getTime()))).toISOString(),
end: new Date(Math.max(...timestamps.map(d => d.getTime()))).toISOString()
} : { start: '', end: '' },
activityTypes: Array.from(activityTypes)
};
});
return {
keyEntities,
entitySummary,
relations,
sessionHistory
};
}
/**
* Smart session continuity - automatically provides context for new chats
*/
async getSessionContinuityContext(query) {
// Get recent context (last 24 hours)
const recentWork = await this.getRecentContext(24, 15);
// Get related work if query provided
let relatedWork;
if (query) {
relatedWork = await this.getRelatedWork(query, 7, 10);
}
// Get historical overview
const historicalOverview = await this.getHistoricalOverview(7, 8);
// Calculate confidence score based on available context
let confidenceScore = 0;
const recommendations = [];
if (recentWork.recentObservations.length > 0) {
confidenceScore += 0.4;
recommendations.push(`Found ${recentWork.recentObservations.length} recent activities`);
}
if (recentWork.activeSessions.length > 0) {
confidenceScore += 0.3;
recommendations.push(`${recentWork.activeSessions.length} active sessions detected`);
}
if (relatedWork && relatedWork.results.length > 0) {
confidenceScore += 0.2;
recommendations.push(`${relatedWork.results.length} related items found`);
}
if (historicalOverview.keyEntities.length > 0) {
confidenceScore += 0.1;
recommendations.push(`${historicalOverview.keyEntities.length} key historical entities available`);
}
return {
recentWork,
relatedWork,
historicalOverview,
confidenceScore,
recommendations
};
}
// Session Summary Management
async loadSessionSummaries() {
try {
const data = await fs.readFile(SESSION_SUMMARIES_FILE_PATH, 'utf-8');
return JSON.parse(data);
}
catch (error) {
return { summaries: [] };
}
}
async saveSessionSummaries(storage) {
await fs.writeFile(SESSION_SUMMARIES_FILE_PATH, JSON.stringify(storage, null, 2));
}
/**
* Summarize and save the current chat session
*/
async saveCurrentSessionSummary(sessionId, summary, startTime, endTime) {
const graph = await this.loadGraph();
const storage = await this.loadSessionSummaries();
// Filter entities and relations for this session
const sessionEntities = graph.entities.filter(e => e.sessionId === sessionId);
const sessionEntityNames = new Set(sessionEntities.map(e => e.name));
const sessionRelations = graph.relations.filter(r => r.sessionId === sessionId ||
(sessionEntityNames.has(r.from) && sessionEntityNames.has(r.to)));
// Collect session observations with activity types
const activityTypes = new Set();
let totalObservations = 0;
for (const entity of sessionEntities) {
for (const obs of entity.observations) {
if (typeof obs === 'object' && obs.sessionId === sessionId) {
totalObservations++;
if (obs.activityType) {
activityTypes.add(obs.activityType);
}
}
}
}
// Find session time boundaries
const sessionObservations = [];
for (const entity of sessionEntities) {
for (const obs of entity.observations) {
if (typeof obs === 'object' && obs.sessionId === sessionId) {
sessionObservations.push(obs);
}
}
}
sessionObservations.sort((a,