@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
341 lines • 15.2 kB
JavaScript
/**
* SQLite-Based Conversation Memory System
* Implements persistent growing memory storage using SQLite as intended
* Provides 24-hour memory, thematic search, and cross-conversation context
*/
import { getDatabase } from '../../storage/unified-database.js';
import { logger } from '../../utils/logging.js';
export class SQLiteMemorySystem {
lastBackupTime = 0;
backupThresholdHours = 24;
constructor() {
logger.info('✅ SQLite memory system using unified database');
}
/**
* Get conversations from the past 24 hours for context
* Uses SQLite curator_truths table as intended for persistent memory
*/
async getRecent24HourContext(userId) {
const db = await getDatabase();
const twentyFourHoursAgo = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
try {
const query = `
SELECT
ct.conversation_id,
kc.question,
ct.curator_output as curator_content,
ct.created_at,
ct.confidence_score,
1.0 as relevance_score
FROM curator_truths ct
JOIN knowledge_conversations kc ON kc.conversation_id = ct.conversation_id
WHERE ct.created_at > ?
ORDER BY ct.confidence_score DESC, ct.created_at DESC
LIMIT 10
`;
const results = await db.all(query, [twentyFourHoursAgo]);
logger.info(`🧠 Retrieved ${results.length} conversations from past 24 hours`);
return results.map(row => ({
...row,
context_type: 'recent_24h',
relevance_score: 1.0
}));
}
catch (error) {
logger.error(`Error fetching 24-hour context from SQLite: ${error instanceof Error ? error.message : 'Unknown error'}`);
return [];
}
}
/**
* Get thematically relevant conversations using SQLite FTS
* Searches curator_truths_fts table for semantic matches
*/
async getThematicContext(query, userId) {
const db = await getDatabase();
try {
// Extract keywords from the query for FTS search
const keywords = this.extractKeywords(query);
const searchTerms = keywords.join(' OR ');
const ftsQuery = `
SELECT
ct.conversation_id,
c.question,
ct.curator_output as curator_content,
ct.created_at,
ct.confidence_score,
bm25(curator_truths_fts) as relevance_score
FROM curator_truths_fts
JOIN curator_truths ct ON ct.conversation_id = curator_truths_fts.conversation_id
JOIN knowledge_conversations kc ON kc.conversation_id = ct.conversation_id
WHERE curator_truths_fts MATCH ?
ORDER BY relevance_score DESC, ct.confidence_score DESC
LIMIT 5
`;
const results = await db.all(ftsQuery, [searchTerms]);
logger.info(`🔍 Found ${results.length} thematically relevant conversations`);
return results.map(row => ({
...row,
context_type: 'thematic'
}));
}
catch (error) {
logger.warn(`FTS search failed, falling back to keyword search: ${error instanceof Error ? error.message : 'Unknown error'}`);
// Fallback to LIKE-based search if FTS fails
return this.fallbackKeywordSearch(query);
}
}
/**
* Fallback keyword search when FTS is not available
*/
async fallbackKeywordSearch(query) {
const db = await getDatabase();
try {
const keywords = this.extractKeywords(query);
const conditions = keywords.map(() => `(ct.curator_output LIKE ? OR kc.question LIKE ?)`).join(' OR ');
const params = keywords.flatMap(keyword => [`%${keyword}%`, `%${keyword}%`]);
const fallbackQuery = `
SELECT
ct.conversation_id,
kc.question,
ct.curator_output as curator_content,
ct.created_at,
ct.confidence_score,
0.5 as relevance_score
FROM curator_truths ct
JOIN knowledge_conversations kc ON kc.conversation_id = ct.conversation_id
WHERE (${conditions})
ORDER BY ct.confidence_score DESC, ct.created_at DESC
LIMIT 5
`;
const results = await db.all(fallbackQuery, params);
logger.info(`📋 Fallback search found ${results.length} relevant conversations`);
return results.map((row) => ({
...row,
context_type: 'thematic'
}));
}
catch (error) {
logger.error(`Error in fallback keyword search: ${error instanceof Error ? error.message : 'Unknown error'}`);
return [];
}
}
/**
* Build complete conversation context from SQLite persistent memory
* This is the core function that implements growing memory storage
*/
async buildFullContext(query, conversationId, userId) {
logger.info(`🧠 Building conversation context for: ${query.substring(0, 50)}...`);
// Get both types of context in parallel from SQLite
const [recent24h, thematic] = await Promise.all([
this.getRecent24HourContext(userId),
this.getThematicContext(query, userId)
]);
// Remove duplicates (in case a conversation appears in both)
const allContext = this.deduplicateContext([...recent24h, ...thematic]);
// Track context sources for this conversation in SQLite
await this.trackContextSources(conversationId, allContext);
// Generate memory summary
const memorySummary = this.generateMemorySummary(allContext);
const context = {
recent_24h: recent24h,
thematic: thematic,
formatted_context: this.formatContextForPrompt(allContext),
source_count: allContext.length,
memory_summary: memorySummary
};
logger.info(`✅ Context built: ${allContext.length} sources (${recent24h.length} recent, ${thematic.length} thematic)`);
return context;
}
/**
* Store curator truth as single source of truth in SQLite
* This implements the growing memory storage as intended
*/
async storeCuratorTruth(conversationId, curatorOutput, confidenceScore, topicSummary) {
const db = await getDatabase();
try {
await db.run(`
INSERT OR REPLACE INTO curator_truths
(conversation_id, curator_output, confidence_score, topic_summary, created_at)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
`, [conversationId, curatorOutput, confidenceScore, topicSummary]);
logger.info(`💾 Stored curator truth for conversation ${conversationId} (confidence: ${(confidenceScore * 100).toFixed(1)}%)`);
// Trigger automatic backup check after storing important conversation data
this.checkAndCreateAutomaticBackup().catch(error => {
logger.debug(`Automatic backup check failed (non-critical): ${error instanceof Error ? error.message : 'Unknown error'}`);
});
}
catch (error) {
logger.error(`Failed to store curator truth in SQLite: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
/**
* Track which context sources were used for a conversation in SQLite
*/
async trackContextSources(conversationId, context) {
if (context.length === 0)
return;
const db = await getDatabase();
try {
const stmt = await db.prepare(`
INSERT OR IGNORE INTO conversation_context
(conversation_id, referenced_conversation_id, relevance_score, context_type, created_at)
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
`);
for (const item of context) {
await stmt.run([
conversationId,
item.conversation_id,
item.relevance_score,
item.context_type
]);
}
await stmt.finalize();
logger.debug(`🔗 Tracked ${context.length} context sources for conversation ${conversationId}`);
}
catch (error) {
logger.error(`Error tracking context sources in SQLite: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
/**
* Format context for inclusion in AI prompts
*/
formatContextForPrompt(context) {
if (context.length === 0) {
return "No relevant conversation history found in persistent memory. Use your own knowledge.";
}
let formatted = "PERSISTENT MEMORY CONTEXT FROM SQLITE:\n\n";
// Group by context type
const recent = context.filter(c => c.context_type === 'recent_24h');
const thematic = context.filter(c => c.context_type === 'thematic');
if (recent.length > 0) {
formatted += "RECENT CONVERSATIONS (Past 24 Hours):\n";
recent.forEach((item, index) => {
const confidence = item.confidence_score ? ` (${(item.confidence_score * 100).toFixed(0)}% confidence)` : '';
formatted += `${index + 1}. Q: ${item.question}\n`;
formatted += ` A: ${item.curator_content.substring(0, 300)}...${confidence}\n`;
formatted += ` (${new Date(item.created_at).toLocaleString()})\n\n`;
});
}
if (thematic.length > 0) {
formatted += "THEMATICALLY RELEVANT CONVERSATIONS:\n";
thematic.forEach((item, index) => {
const confidence = item.confidence_score ? ` (${(item.confidence_score * 100).toFixed(0)}% confidence)` : '';
formatted += `${index + 1}. Q: ${item.question}\n`;
formatted += ` A: ${item.curator_content.substring(0, 300)}...${confidence}\n`;
formatted += ` (Relevance: ${(item.relevance_score * 100).toFixed(0)}%)\n\n`;
});
}
formatted += "END PERSISTENT MEMORY CONTEXT\n\n";
formatted += "INSTRUCTIONS: Use this SQLite-stored context intelligently to inform your response. Build upon past curator truths when relevant.";
return formatted;
}
/**
* Generate a summary of the memory context
*/
generateMemorySummary(context) {
if (context.length === 0) {
return "No persistent memory context available.";
}
const recentCount = context.filter(c => c.context_type === 'recent_24h').length;
const thematicCount = context.filter(c => c.context_type === 'thematic').length;
const avgConfidence = context
.filter(c => c.confidence_score)
.reduce((sum, c) => sum + (c.confidence_score || 0), 0) / context.length;
return `SQLite Memory: ${context.length} sources (${recentCount} recent, ${thematicCount} thematic), avg confidence: ${(avgConfidence * 100).toFixed(0)}%`;
}
/**
* Remove duplicate conversations from context
*/
deduplicateContext(context) {
const seen = new Set();
return context.filter(item => {
if (seen.has(item.conversation_id)) {
return false;
}
seen.add(item.conversation_id);
return true;
});
}
/**
* Extract meaningful keywords from a query
*/
extractKeywords(query) {
// Simple keyword extraction - can be enhanced with NLP
const stopWords = new Set([
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'have',
'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should',
'what', 'how', 'why', 'when', 'where', 'who', 'which'
]);
return query
.toLowerCase()
.replace(/[^\w\s]/g, ' ')
.split(/\s+/)
.filter(word => word.length > 2 && !stopWords.has(word))
.slice(0, 10); // Limit to top 10 keywords
}
/**
* Get memory statistics for debugging
*/
async getMemoryStats() {
const db = await getDatabase();
try {
const totalConversations = await db.get('SELECT COUNT(*) as count FROM knowledge_conversations');
const totalCuratorTruths = await db.get('SELECT COUNT(*) as count FROM curator_truths');
const totalContextLinks = await db.get('SELECT COUNT(*) as count FROM conversation_context');
const avgConfidence = await db.get('SELECT AVG(confidence_score) as avg FROM curator_truths');
const recent24h = await db.get(`
SELECT COUNT(*) as count FROM curator_truths
WHERE created_at > datetime('now', '-24 hours')
`);
return {
total_conversations: totalConversations?.count || 0,
total_curator_truths: totalCuratorTruths?.count || 0,
total_context_links: totalContextLinks?.count || 0,
average_confidence: avgConfidence?.avg || 0,
recent_24h_conversations: recent24h?.count || 0,
database_path: 'unified database (hive-ai.db)'
};
}
catch (error) {
logger.error(`Error getting memory stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
return { error: error instanceof Error ? error.message : 'Unknown error' };
}
}
/**
* Check if automatic backup should be created and trigger if needed
*/
async checkAndCreateAutomaticBackup() {
const now = Date.now();
const hoursSinceLastBackup = (now - this.lastBackupTime) / (1000 * 60 * 60);
if (hoursSinceLastBackup >= this.backupThresholdHours) {
try {
// Import backup manager dynamically to avoid circular dependency
const { SQLiteBackupManager } = await import('./sqlite-backup-manager.js');
const backupManager = new SQLiteBackupManager(this);
logger.info('🔄 Creating automatic backup...');
const result = await backupManager.createBackup('full');
if (result.success) {
this.lastBackupTime = now;
logger.info('✅ Automatic backup created successfully');
}
else {
logger.warn(`⚠️ Automatic backup failed: ${result.error}`);
}
}
catch (error) {
logger.warn(`⚠️ Failed to create automatic backup: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
}
/**
* Close database connection
*/
async close() {
// No need to close - unified database manages connections
logger.info('🔒 SQLite memory system using unified database');
}
}
// Singleton instance for global use
export const sqliteMemory = new SQLiteMemorySystem();
//# sourceMappingURL=sqlite-memory.js.map