codecrucible-synth
Version:
Production-Ready AI Development Platform with Multi-Voice Synthesis, Smithery MCP Integration, Enterprise Security, and Zero-Timeout Reliability
613 lines • 21.6 kB
JavaScript
import * as sqlite3 from 'sqlite3';
import { open } from 'sqlite';
import * as path from 'path';
import * as fs from 'fs/promises';
import { logger } from '../logger.js';
/**
* Persistent conversation storage with semantic search capabilities
* Provides SQL-based storage with embeddings for intelligent retrieval
*/
export class ConversationStore {
db;
dbPath;
initialized = false;
constructor(workspaceRoot) {
const codecrucibleDir = path.join(workspaceRoot, '.codecrucible');
this.dbPath = path.join(codecrucibleDir, 'conversations.db');
logger.info('Conversation store initialized', { dbPath: this.dbPath });
}
/**
* Initialize database and create tables
*/
async initialize() {
if (this.initialized)
return;
try {
// Ensure directory exists
await fs.mkdir(path.dirname(this.dbPath), { recursive: true });
// Open database
this.db = await open({
filename: this.dbPath,
driver: sqlite3.Database,
});
// Create tables
await this.createTables();
// Create indexes for performance
await this.createIndexes();
this.initialized = true;
logger.info('Conversation store database initialized');
}
catch (error) {
logger.error('Failed to initialize conversation store:', error);
throw error;
}
}
/**
* Store interaction with optional semantic embedding
*/
async storeInteraction(prompt, response, context, sessionId, userFeedback) {
await this.ensureInitialized();
const id = this.generateId();
const timestamp = Date.now();
const contextHash = this.hashContext(context);
const topics = this.extractTopics(prompt, response.synthesis || '');
// Generate embedding for semantic search (simplified for now)
const embedding = await this.generateEmbedding(prompt + ' ' + response.synthesis);
const interaction = {
id,
sessionId,
timestamp,
prompt,
response: response.synthesis || '',
voicesUsed: response.voicesUsed,
confidence: response.confidence || 0,
latency: response.latency || 0,
userFeedback,
topics,
contextHash,
embedding,
metadata: {
modelUsed: response.modelUsed,
reasoning: typeof response.reasoning === 'object' &&
response.reasoning &&
'steps' in response.reasoning &&
Array.isArray(response.reasoning.steps)
? response.reasoning.steps.length
: 0,
promptTokens: prompt.length, // Simplified token count
responseTokens: (response.synthesis || '').length,
},
};
try {
await this.db.run(`
INSERT INTO interactions (
id, session_id, timestamp, prompt, response, voices_used,
confidence, latency, user_feedback, topics, context_hash,
embedding, metadata
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, [
interaction.id,
interaction.sessionId,
interaction.timestamp,
interaction.prompt,
interaction.response,
JSON.stringify(interaction.voicesUsed),
interaction.confidence,
interaction.latency,
interaction.userFeedback,
JSON.stringify(interaction.topics),
interaction.contextHash,
JSON.stringify(interaction.embedding),
JSON.stringify(interaction.metadata),
]);
// Update session statistics
await this.updateSessionStats(sessionId, interaction);
logger.debug('Stored interaction', {
id: interaction.id,
sessionId: interaction.sessionId,
topics: interaction.topics.length,
confidence: interaction.confidence,
});
return id;
}
catch (error) {
logger.error('Failed to store interaction:', error);
throw error;
}
}
/**
* Search interactions with text and semantic similarity
*/
async searchInteractions(query) {
await this.ensureInitialized();
let sql = `
SELECT i.*, s.workspace_root
FROM interactions i
LEFT JOIN sessions s ON i.session_id = s.id
WHERE 1=1
`;
const params = [];
// Add filters
if (query.text) {
sql += ` AND (i.prompt LIKE ? OR i.response LIKE ?)`;
const searchPattern = `%${query.text}%`;
params.push(searchPattern, searchPattern);
}
if (query.sessionId) {
sql += ` AND i.session_id = ?`;
params.push(query.sessionId);
}
if (query.timeRange) {
sql += ` AND i.timestamp BETWEEN ? AND ?`;
params.push(query.timeRange.start, query.timeRange.end);
}
if (query.topics && query.topics.length > 0) {
sql += ` AND (${query.topics.map(() => 'i.topics LIKE ?').join(' OR ')})`;
query.topics.forEach(topic => params.push(`%"${topic}"%`));
}
if (query.voices && query.voices.length > 0) {
sql += ` AND (${query.voices.map(() => 'i.voices_used LIKE ?').join(' OR ')})`;
query.voices.forEach(voice => params.push(`%"${voice}"%`));
}
if (query.minConfidence !== undefined) {
sql += ` AND i.confidence >= ?`;
params.push(query.minConfidence);
}
// Order by timestamp descending
sql += ` ORDER BY i.timestamp DESC`;
// Add pagination
if (query.limit) {
sql += ` LIMIT ?`;
params.push(query.limit);
if (query.offset) {
sql += ` OFFSET ?`;
params.push(query.offset);
}
}
try {
const rows = await this.db.all(sql, params);
const interactions = rows.map(row => this.mapRowToInteraction(row));
// Get total count
const countSql = sql
.replace(/SELECT i\.\*, s\.workspace_root/, 'SELECT COUNT(*)')
.replace(/ORDER BY i\.timestamp DESC.*$/, '');
const countResult = await this.db.get(countSql, params.slice(0, -2)); // Remove LIMIT/OFFSET params
const total = countResult['COUNT(*)'] || 0;
// Calculate relevance scores if text search
let relevanceScores;
if (query.text) {
relevanceScores = await this.calculateRelevanceScores(query.text, interactions);
}
logger.debug('Search completed', {
query: query.text?.substring(0, 50),
results: interactions.length,
total,
});
return {
interactions,
total,
relevanceScores,
};
}
catch (error) {
logger.error('Search failed:', error);
throw error;
}
}
/**
* Get conversation session by ID
*/
async getSession(sessionId) {
await this.ensureInitialized();
try {
const row = await this.db.get('SELECT * FROM sessions WHERE id = ?', [sessionId]);
if (!row)
return null;
return {
id: row.id,
startTime: row.start_time,
endTime: row.end_time,
totalInteractions: row.total_interactions,
averageConfidence: row.average_confidence,
topics: JSON.parse(row.topics || '[]'),
workspaceRoot: row.workspace_root,
userAgent: row.user_agent,
};
}
catch (error) {
logger.error(`Failed to get session ${sessionId}:`, error);
return null;
}
}
/**
* Start a new conversation session
*/
async startSession(workspaceRoot, userAgent) {
await this.ensureInitialized();
const sessionId = this.generateId();
const startTime = Date.now();
try {
await this.db.run(`
INSERT INTO sessions (id, start_time, workspace_root, user_agent, total_interactions, average_confidence, topics)
VALUES (?, ?, ?, ?, 0, 0, '[]')
`, [sessionId, startTime, workspaceRoot, userAgent]);
logger.info('Started conversation session', { sessionId, workspaceRoot });
return sessionId;
}
catch (error) {
logger.error('Failed to start session:', error);
throw error;
}
}
/**
* End a conversation session
*/
async endSession(sessionId) {
await this.ensureInitialized();
try {
await this.db.run(`
UPDATE sessions
SET end_time = ?
WHERE id = ?
`, [Date.now(), sessionId]);
logger.info('Ended conversation session', { sessionId });
}
catch (error) {
logger.error(`Failed to end session ${sessionId}:`, error);
}
}
/**
* Get conversation analytics
*/
async getAnalytics(timeRange) {
await this.ensureInitialized();
try {
let whereClause = '';
const params = [];
if (timeRange) {
whereClause = 'WHERE timestamp BETWEEN ? AND ?';
params.push(timeRange.start, timeRange.end);
}
// Total interactions and sessions
const totals = await this.db.get(`
SELECT
COUNT(*) as total_interactions,
COUNT(DISTINCT session_id) as total_sessions,
AVG(confidence) as average_confidence,
AVG(latency) as average_latency
FROM interactions
${whereClause}
`, params);
// Top topics
const topicsQuery = await this.db.all(`
SELECT topics
FROM interactions
${whereClause}
ORDER BY timestamp DESC
LIMIT 1000
`, params);
const topicCounts = new Map();
topicsQuery.forEach(row => {
try {
const topics = JSON.parse(row.topics || '[]');
topics.forEach((topic) => {
topicCounts.set(topic, (topicCounts.get(topic) || 0) + 1);
});
}
catch (error) {
// Ignore JSON parse errors
}
});
const topTopics = Array.from(topicCounts.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 10)
.map(([topic, count]) => ({ topic, count }));
// Voice usage
const voicesQuery = await this.db.all(`
SELECT voices_used
FROM interactions
${whereClause}
ORDER BY timestamp DESC
LIMIT 1000
`, params);
const voiceCounts = new Map();
voicesQuery.forEach(row => {
try {
const voices = JSON.parse(row.voices_used || '[]');
voices.forEach((voice) => {
voiceCounts.set(voice, (voiceCounts.get(voice) || 0) + 1);
});
}
catch (error) {
// Ignore JSON parse errors
}
});
const voiceUsage = Array.from(voiceCounts.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 10)
.map(([voice, count]) => ({ voice, count }));
// Daily activity (last 30 days)
const thirtyDaysAgo = Date.now() - 30 * 24 * 60 * 60 * 1000;
const dailyQuery = await this.db.all(`
SELECT
DATE(timestamp / 1000, 'unixepoch') as date,
COUNT(*) as interactions
FROM interactions
WHERE timestamp > ?
GROUP BY DATE(timestamp / 1000, 'unixepoch')
ORDER BY date
`, [thirtyDaysAgo]);
const dailyActivity = dailyQuery.map(row => ({
date: row.date,
interactions: row.interactions,
}));
return {
totalInteractions: totals.total_interactions || 0,
totalSessions: totals.total_sessions || 0,
averageConfidence: totals.average_confidence || 0,
averageLatency: totals.average_latency || 0,
topTopics,
voiceUsage,
dailyActivity,
};
}
catch (error) {
logger.error('Failed to get analytics:', error);
throw error;
}
}
/**
* Update user feedback for an interaction
*/
async updateFeedback(interactionId, feedback) {
await this.ensureInitialized();
try {
await this.db.run(`
UPDATE interactions
SET user_feedback = ?
WHERE id = ?
`, [feedback, interactionId]);
logger.debug('Updated interaction feedback', { interactionId, feedback });
}
catch (error) {
logger.error(`Failed to update feedback for ${interactionId}:`, error);
throw error;
}
}
/**
* Create database tables
*/
async createTables() {
// Sessions table
await this.db.exec(`
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
start_time INTEGER NOT NULL,
end_time INTEGER,
total_interactions INTEGER DEFAULT 0,
average_confidence REAL DEFAULT 0,
topics TEXT DEFAULT '[]',
workspace_root TEXT,
user_agent TEXT
)
`);
// Interactions table
await this.db.exec(`
CREATE TABLE IF NOT EXISTS interactions (
id TEXT PRIMARY KEY,
session_id TEXT NOT NULL,
timestamp INTEGER NOT NULL,
prompt TEXT NOT NULL,
response TEXT NOT NULL,
voices_used TEXT DEFAULT '[]',
confidence REAL DEFAULT 0,
latency INTEGER DEFAULT 0,
user_feedback TEXT,
topics TEXT DEFAULT '[]',
context_hash TEXT,
embedding TEXT,
metadata TEXT DEFAULT '{}',
FOREIGN KEY (session_id) REFERENCES sessions (id)
)
`);
}
/**
* Create database indexes for performance
*/
async createIndexes() {
const indexes = [
'CREATE INDEX IF NOT EXISTS idx_interactions_session_id ON interactions(session_id)',
'CREATE INDEX IF NOT EXISTS idx_interactions_timestamp ON interactions(timestamp)',
'CREATE INDEX IF NOT EXISTS idx_interactions_confidence ON interactions(confidence)',
'CREATE INDEX IF NOT EXISTS idx_interactions_topics ON interactions(topics)',
'CREATE INDEX IF NOT EXISTS idx_sessions_start_time ON sessions(start_time)',
'CREATE INDEX IF NOT EXISTS idx_sessions_workspace ON sessions(workspace_root)',
];
for (const indexSql of indexes) {
await this.db.exec(indexSql);
}
}
/**
* Map database row to StoredInteraction
*/
mapRowToInteraction(row) {
return {
id: row.id,
sessionId: row.session_id,
timestamp: row.timestamp,
prompt: row.prompt,
response: row.response,
voicesUsed: JSON.parse(row.voices_used || '[]'),
confidence: row.confidence,
latency: row.latency,
userFeedback: row.user_feedback,
topics: JSON.parse(row.topics || '[]'),
contextHash: row.context_hash,
embedding: JSON.parse(row.embedding || '[]'),
metadata: JSON.parse(row.metadata || '{}'),
};
}
/**
* Update session statistics
*/
async updateSessionStats(sessionId, interaction) {
const session = await this.getSession(sessionId);
if (!session)
return;
const newTotal = session.totalInteractions + 1;
const newAverage = (session.averageConfidence * session.totalInteractions + interaction.confidence) / newTotal;
// Merge topics
const existingTopics = new Set(session.topics);
interaction.topics.forEach(topic => existingTopics.add(topic));
const mergedTopics = Array.from(existingTopics);
await this.db.run(`
UPDATE sessions
SET total_interactions = ?, average_confidence = ?, topics = ?
WHERE id = ?
`, [newTotal, newAverage, JSON.stringify(mergedTopics), sessionId]);
}
/**
* Generate simple embedding for semantic search
*/
async generateEmbedding(text) {
// This is a simplified embedding generation
// In a real implementation, you would use a proper embedding model
const words = text.toLowerCase().split(/\s+/);
const embedding = new Array(100).fill(0);
// Simple hash-based embedding
words.forEach((word, index) => {
const hash = this.simpleHash(word);
embedding[hash % 100] += 1;
});
// Normalize
const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
if (magnitude > 0) {
return embedding.map(val => val / magnitude);
}
return embedding;
}
/**
* Calculate relevance scores for search results
*/
async calculateRelevanceScores(query, interactions) {
const queryEmbedding = await this.generateEmbedding(query);
return interactions.map(interaction => {
if (!interaction.embedding || interaction.embedding.length === 0) {
// Fallback to text similarity
return this.calculateTextSimilarity(query, interaction.prompt + ' ' + interaction.response);
}
// Calculate cosine similarity
return this.cosineSimilarity(queryEmbedding, interaction.embedding);
});
}
/**
* Calculate cosine similarity between vectors
*/
cosineSimilarity(a, b) {
if (a.length !== b.length)
return 0;
const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0);
const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0));
const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0));
if (magnitudeA === 0 || magnitudeB === 0)
return 0;
return dotProduct / (magnitudeA * magnitudeB);
}
/**
* Calculate text similarity (fallback)
*/
calculateTextSimilarity(query, text) {
const queryWords = new Set(query.toLowerCase().split(/\s+/));
const textWords = text.toLowerCase().split(/\s+/);
const matches = textWords.filter(word => queryWords.has(word)).length;
return matches / Math.max(queryWords.size, textWords.length);
}
/**
* Extract topics from text
*/
extractTopics(prompt, response) {
const text = `${prompt} ${response}`.toLowerCase();
const topics = new Set();
// Programming languages
const languages = [
'javascript',
'typescript',
'python',
'java',
'rust',
'go',
'cpp',
'c#',
'php',
];
languages.forEach(lang => {
if (text.includes(lang))
topics.add(lang);
});
// Technologies
const technologies = [
'react',
'vue',
'angular',
'docker',
'kubernetes',
'git',
'database',
'api',
];
technologies.forEach(tech => {
if (text.includes(tech))
topics.add(tech);
});
return Array.from(topics);
}
/**
* Generate hash for context
*/
hashContext(context) {
const str = JSON.stringify({
guidance: context.guidance?.substring(0, 100),
preferences: context.preferences,
patterns: context.patterns.map(p => p.name),
});
return this.simpleHash(str).toString();
}
/**
* Simple hash function
*/
simpleHash(str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
const char = str.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash = hash & hash; // Convert to 32-bit integer
}
return Math.abs(hash);
}
/**
* Generate unique ID
*/
generateId() {
return `${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
/**
* Ensure database is initialized
*/
async ensureInitialized() {
if (!this.initialized) {
await this.initialize();
}
}
/**
* Close database connection
*/
async dispose() {
if (this.db) {
await this.db.close();
this.db = undefined;
this.initialized = false;
logger.info('Conversation store disposed');
}
}
}
//# sourceMappingURL=conversation-store.js.map