@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
1,009 lines (962 loc) • 119 kB
JavaScript
/**
* Unified Database Manager - Single Source of Truth
*
* Consolidates all 6 separate SQLite databases into one normalized database
* with proper referential integrity and ACID compliance.
*/
import { open } from 'sqlite';
import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs';
import { structuredLogger } from '../tools/structured-logger.js';
// Multiple database paths with fallbacks for Windows enterprise environments
const STORAGE_PATHS = [
path.join(os.homedir(), '.hive-ai'),
path.join(os.tmpdir(), `hive-ai-${(os.userInfo().username || 'user').replace(/[^a-zA-Z0-9]/g, '_')}`),
...(process.platform === 'win32' ? [
path.join(process.env.LOCALAPPDATA || os.tmpdir(), 'hive-ai'),
path.join(process.env.APPDATA || os.tmpdir(), 'hive-ai'),
] : [])
];
// Test and select the first writable directory
function getWritableStoragePath() {
for (const testPath of STORAGE_PATHS) {
try {
if (!fs.existsSync(testPath)) {
fs.mkdirSync(testPath, { recursive: true, mode: 0o755 });
}
fs.accessSync(testPath, fs.constants.W_OK);
return testPath;
}
catch (error) {
console.warn(`⚠️ Storage path not writable: ${testPath}`);
continue;
}
}
throw new Error('No writable storage path found. Check permissions or contact IT support.');
}
// Initialize storage paths
let DB_DIR;
let DB_PATH;
// Initialize storage location safely
try {
DB_DIR = getWritableStoragePath();
DB_PATH = path.join(DB_DIR, 'hive-ai.db');
console.log(`📁 Using storage directory: ${DB_DIR}`);
}
catch (error) {
console.error('❌ Failed to initialize storage:', error);
// Fallback to memory-only mode for extremely restricted environments
DB_DIR = ':memory:';
DB_PATH = ':memory:';
console.warn('⚠️ Running in memory-only mode - data will not persist');
}
// Directory creation will be handled during initialization to avoid module-level side effects
let db;
let isInitialized = false;
/**
* Initialize the unified database with complete normalized schema
* Uses singleton pattern to prevent multiple initializations
*/
export async function initializeUnifiedDatabase() {
// Return early if already initialized
if (isInitialized && db) {
return true;
}
try {
// Create directory if needed with proper error handling
try {
if (!fs.existsSync(DB_DIR)) {
// Windows-specific: Use more permissive mode
const mode = process.platform === 'win32' ? 0o755 : 0o755;
fs.mkdirSync(DB_DIR, { recursive: true, mode });
}
// Test if directory is writable
fs.accessSync(DB_DIR, fs.constants.W_OK);
}
catch (dirError) {
console.error('❌ Cannot create or write to database directory:', DB_DIR);
console.error('Error:', dirError.message);
console.error(`Platform: ${process.platform}`);
console.error(`User: ${process.env.USER || process.env.USERNAME || 'unknown'}`);
// Windows-specific troubleshooting
if (process.platform === 'win32') {
console.error('💡 Windows troubleshooting:');
console.error(' • Try running as Administrator');
console.error(' • Check antivirus is not blocking file creation');
console.error(' • Ensure %USERPROFILE% is writable');
}
throw new Error(`Database directory not accessible: ${dirError.message}`);
}
// Open database connection with retry logic for Windows
const sqlite3Driver = await import('sqlite3');
let retries = 3;
while (retries > 0) {
try {
db = await open({
filename: DB_PATH,
driver: sqlite3Driver.default.Database,
});
break;
}
catch (openError) {
retries--;
if (retries === 0) {
console.error('❌ Failed to open database after 3 attempts');
console.error('Error:', openError.message);
if (process.platform === 'win32' && openError.message.includes('lock')) {
console.error('💡 Windows database lock detected - try closing other Hive instances');
}
throw openError;
}
// Wait a bit before retrying (Windows file locking)
await new Promise(resolve => setTimeout(resolve, 100));
}
}
// Test database write access
try {
await db.run('SELECT 1');
}
catch (testError) {
if (testError.code === 'SQLITE_READONLY') {
console.error('❌ Database file is read-only:', DB_PATH);
if (process.platform === 'win32') {
console.error('💡 Windows: Check file properties and remove read-only flag');
}
throw new Error('Database is read-only - cannot initialize properly');
}
throw testError;
}
// Enable foreign key constraints
await db.exec('PRAGMA foreign_keys = ON');
// Create complete normalized schema
await db.exec(`
-- Users and authentication
CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY,
email TEXT UNIQUE,
license_key TEXT,
tier TEXT DEFAULT 'FREE',
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Configuration storage (API keys, settings)
CREATE TABLE IF NOT EXISTS configurations (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
encrypted BOOLEAN DEFAULT 0,
user_id TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
);
-- OpenRouter providers (consolidated from openrouter-data.db)
CREATE TABLE IF NOT EXISTS openrouter_providers (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
model_count INTEGER DEFAULT 0,
average_cost REAL DEFAULT 0.0,
capabilities TEXT DEFAULT '[]',
last_updated TEXT NOT NULL,
is_active BOOLEAN DEFAULT 1,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- OpenRouter models (consolidated from openrouter-data.db)
CREATE TABLE IF NOT EXISTS openrouter_models (
internal_id INTEGER PRIMARY KEY AUTOINCREMENT, -- Stable rowid, NEVER changes
openrouter_id TEXT NOT NULL UNIQUE, -- Current OpenRouter model ID (can change)
name TEXT NOT NULL,
provider_id TEXT NOT NULL,
provider_name TEXT NOT NULL,
description TEXT DEFAULT '',
capabilities TEXT DEFAULT '[]',
pricing_input REAL DEFAULT 0.0,
pricing_output REAL DEFAULT 0.0,
pricing_image REAL,
pricing_request REAL,
context_window INTEGER DEFAULT 4096,
created_at INTEGER NOT NULL,
input_modalities TEXT DEFAULT '["text"]',
output_modalities TEXT DEFAULT '["text"]',
is_active BOOLEAN DEFAULT 1,
last_updated TEXT NOT NULL,
FOREIGN KEY (provider_id) REFERENCES openrouter_providers(id)
);
-- Pipeline profiles (consolidated from secure-pipeline.db)
CREATE TABLE IF NOT EXISTS pipeline_profiles (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
user_id TEXT,
is_default BOOLEAN DEFAULT 0,
-- New stable foreign key references (bulletproof against model ID changes)
generator_model_internal_id INTEGER NOT NULL,
generator_temperature REAL DEFAULT 0.7,
refiner_model_internal_id INTEGER NOT NULL,
refiner_temperature REAL DEFAULT 0.7,
validator_model_internal_id INTEGER NOT NULL,
validator_temperature REAL DEFAULT 0.7,
curator_model_internal_id INTEGER NOT NULL,
curator_temperature REAL DEFAULT 0.7,
-- Legacy columns for migration (will be removed after migration)
generator_provider TEXT,
generator_model TEXT,
refiner_provider TEXT,
refiner_model TEXT,
validator_provider TEXT,
validator_model TEXT,
curator_provider TEXT,
curator_model TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (generator_model_internal_id) REFERENCES openrouter_models(internal_id),
FOREIGN KEY (refiner_model_internal_id) REFERENCES openrouter_models(internal_id),
FOREIGN KEY (validator_model_internal_id) REFERENCES openrouter_models(internal_id),
FOREIGN KEY (curator_model_internal_id) REFERENCES openrouter_models(internal_id)
);
-- Consensus profiles (existing from hive-ai-context.db)
CREATE TABLE IF NOT EXISTS consensus_profiles (
id TEXT PRIMARY KEY,
profile_name TEXT NOT NULL UNIQUE,
pipeline_profile_id TEXT,
generator_model TEXT NOT NULL,
refiner_model TEXT NOT NULL,
validator_model TEXT NOT NULL,
curator_model TEXT NOT NULL,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (pipeline_profile_id) REFERENCES pipeline_profiles(id)
);
-- Conversations (existing from hive-ai-context.db)
CREATE TABLE IF NOT EXISTS conversations (
id TEXT PRIMARY KEY,
user_id TEXT,
consensus_profile_id TEXT,
total_cost REAL DEFAULT 0,
input_tokens INTEGER DEFAULT 0,
output_tokens INTEGER DEFAULT 0,
start_time TEXT,
end_time TEXT,
success_rate TEXT DEFAULT "100",
quality_score REAL DEFAULT 0.95,
consensus_improvement INTEGER DEFAULT 15,
confidence_level TEXT DEFAULT "High",
success INTEGER DEFAULT 1,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (consensus_profile_id) REFERENCES consensus_profiles(id)
);
-- Messages (existing from hive-ai-context.db)
CREATE TABLE IF NOT EXISTS messages (
id TEXT PRIMARY KEY,
conversation_id TEXT NOT NULL,
role TEXT NOT NULL,
content TEXT NOT NULL,
stage TEXT,
model_used TEXT,
timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Usage tracking (replaces JSON files)
CREATE TABLE IF NOT EXISTS usage_records (
id TEXT PRIMARY KEY,
user_id TEXT,
conversation_id TEXT,
action_type TEXT NOT NULL, -- 'conversation', 'model_call', 'sync'
cost REAL DEFAULT 0.0,
tokens_input INTEGER DEFAULT 0,
tokens_output INTEGER DEFAULT 0,
model_used TEXT,
timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id),
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Budget tracking (replaces budget.json)
CREATE TABLE IF NOT EXISTS budget_limits (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
period_type TEXT NOT NULL, -- 'daily', 'monthly', 'yearly'
limit_amount REAL NOT NULL,
current_spent REAL DEFAULT 0.0,
period_start TEXT NOT NULL,
period_end TEXT NOT NULL,
is_active BOOLEAN DEFAULT 1,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
);
-- Sync metadata (from openrouter-data.db)
CREATE TABLE IF NOT EXISTS sync_metadata (
id TEXT PRIMARY KEY,
sync_type TEXT NOT NULL,
started_at TEXT NOT NULL,
completed_at TEXT,
status TEXT NOT NULL,
providers_synced INTEGER DEFAULT 0,
models_synced INTEGER DEFAULT 0,
error_message TEXT,
next_sync_due TEXT NOT NULL,
rankings_last_synced TEXT,
provider_performance_synced TEXT,
intelligence_version TEXT DEFAULT '1.7.0',
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Settings (global app settings)
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
description TEXT,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Consensus settings (for active profile tracking)
CREATE TABLE IF NOT EXISTS consensus_settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Legacy compatibility tables (for gradual migration)
CREATE TABLE IF NOT EXISTS conversation_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL UNIQUE,
user_id TEXT,
timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id),
FOREIGN KEY (user_id) REFERENCES users(id)
);
CREATE TABLE IF NOT EXISTS consensus_settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Knowledge Database Tables (from hive-ai-knowledge.db)
-- Extended conversation data with questions, answers, and source-of-truth
CREATE TABLE IF NOT EXISTS knowledge_conversations (
id TEXT PRIMARY KEY,
conversation_id TEXT, -- Links to main conversations table
question TEXT NOT NULL,
final_answer TEXT NOT NULL,
source_of_truth TEXT NOT NULL,
conversation_context TEXT,
profile_id TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
last_updated TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
CREATE TABLE IF NOT EXISTS curator_truths (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL UNIQUE,
curator_output TEXT NOT NULL,
confidence_score REAL NOT NULL,
topic_summary TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
CREATE TABLE IF NOT EXISTS conversation_context (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
referenced_conversation_id TEXT NOT NULL,
relevance_score REAL DEFAULT 1.0,
context_type TEXT CHECK (context_type IN ('recent_24h', 'thematic', 'direct_reference')),
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id),
FOREIGN KEY (referenced_conversation_id) REFERENCES conversations(id)
);
CREATE TABLE IF NOT EXISTS conversation_topics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
topic TEXT NOT NULL,
relevance_score REAL DEFAULT 1.0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
CREATE TABLE IF NOT EXISTS conversation_keywords (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
keyword TEXT NOT NULL,
frequency INTEGER DEFAULT 1,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Improvement patterns table for tracking what each stage typically improves
CREATE TABLE IF NOT EXISTS improvement_patterns (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
from_stage TEXT NOT NULL,
to_stage TEXT NOT NULL,
improvement_type TEXT NOT NULL,
description TEXT NOT NULL,
question_type TEXT NOT NULL,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Conversation thread tracking for follow-up detection
CREATE TABLE IF NOT EXISTS conversation_threads (
id INTEGER PRIMARY KEY AUTOINCREMENT,
child_conversation_id TEXT NOT NULL,
parent_conversation_id TEXT NOT NULL,
thread_type TEXT CHECK (thread_type IN ('follow_up', 'clarification', 'list_reference', 'continuation')),
reference_text TEXT, -- The text that indicated this was a follow-up
confidence_score REAL DEFAULT 1.0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (child_conversation_id) REFERENCES conversations(id),
FOREIGN KEY (parent_conversation_id) REFERENCES conversations(id)
);
-- Pending sync table for tracking conversations awaiting server verification
-- Note: No foreign keys as this is a temporary queue, not permanent data
CREATE TABLE IF NOT EXISTS pending_sync (
conversation_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
question_hash TEXT NOT NULL,
conversation_token TEXT NOT NULL,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
sync_attempts INTEGER DEFAULT 0,
last_attempt_at TEXT,
error_message TEXT
);
-- ===== DYNAMIC CONSENSUS & OPENROUTER INTELLIGENCE TABLES =====
-- Provider performance tracking for intelligent routing
CREATE TABLE IF NOT EXISTS provider_performance (
id INTEGER PRIMARY KEY AUTOINCREMENT,
model_internal_id INTEGER NOT NULL,
provider_name TEXT NOT NULL,
routing_variant TEXT, -- :nitro, :floor, :online, :free, :default
avg_latency_ms INTEGER,
success_rate REAL DEFAULT 1.0,
cost_per_token REAL,
uptime_24h REAL DEFAULT 1.0,
error_rate REAL DEFAULT 0.0,
last_success_at TEXT,
last_failure_at TEXT,
measured_at TEXT DEFAULT CURRENT_TIMESTAMP,
measurement_period_hours INTEGER DEFAULT 24,
FOREIGN KEY (model_internal_id) REFERENCES openrouter_models(internal_id)
);
-- OpenRouter programming model rankings
CREATE TABLE IF NOT EXISTS model_rankings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
model_internal_id INTEGER NOT NULL,
ranking_source TEXT NOT NULL, -- 'openrouter_programming_weekly', 'openrouter_programming_monthly'
rank_position INTEGER NOT NULL,
usage_percentage REAL NOT NULL,
relative_score REAL, -- Normalized score for comparison
period_start TEXT NOT NULL,
period_end TEXT NOT NULL,
collected_at TEXT DEFAULT CURRENT_TIMESTAMP,
data_quality TEXT DEFAULT 'scraped', -- 'scraped', 'api', 'estimated'
FOREIGN KEY (model_internal_id) REFERENCES openrouter_models(internal_id)
);
-- Consensus effectiveness measurement and A/B testing
CREATE TABLE IF NOT EXISTS consensus_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
baseline_model TEXT NOT NULL, -- Single model used for comparison
baseline_result TEXT NOT NULL, -- Single model output
consensus_result TEXT NOT NULL, -- 4-stage consensus output
improvement_score REAL NOT NULL, -- 0-1 score: consensus improvement over baseline
quality_metrics TEXT NOT NULL, -- JSON: correctness, completeness, architecture, etc.
cost_comparison TEXT NOT NULL, -- JSON: single model cost vs consensus cost
time_comparison TEXT NOT NULL, -- JSON: single model time vs consensus time
user_rating INTEGER, -- 1-5 user satisfaction rating
question_complexity TEXT NOT NULL, -- minimal/basic/production
question_category TEXT, -- coding/architecture/debugging/etc.
effectiveness_notes TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Cost optimization and budget tracking per conversation
CREATE TABLE IF NOT EXISTS cost_analytics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
total_cost REAL NOT NULL,
cost_per_stage TEXT NOT NULL, -- JSON: {generator: 0.05, refiner: 0.03, etc.}
tokens_per_stage TEXT NOT NULL, -- JSON: {generator: {input: 150, output: 300}, etc.}
routing_optimizations TEXT, -- JSON: {stage: variant_used, savings: amount}
budget_threshold REAL,
budget_utilized_percentage REAL,
cost_efficiency_score REAL, -- quality improvement per dollar spent
optimization_opportunities TEXT, -- JSON: suggestions for cost reduction
provider_breakdown TEXT, -- JSON: cost per provider used
savings_achieved REAL DEFAULT 0.0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- OpenRouter feature utilization tracking
CREATE TABLE IF NOT EXISTS feature_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
features_used TEXT NOT NULL, -- JSON: {streaming: true, json_mode: false, tools: ['web_search']}
routing_variants_used TEXT, -- JSON: {generator: ':nitro', refiner: ':default', etc.}
fallback_events INTEGER DEFAULT 0,
provider_switches INTEGER DEFAULT 0,
rate_limit_hits INTEGER DEFAULT 0,
error_recoveries INTEGER DEFAULT 0,
performance_metrics TEXT, -- JSON: {avg_latency: 1200, total_tokens: 1500, etc.}
advanced_features_used TEXT, -- JSON: {function_calling: false, multimodal: false}
cost_optimizations_applied TEXT, -- JSON: optimization strategies used
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Dynamic profile template definitions
CREATE TABLE IF NOT EXISTS profile_templates (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
description TEXT NOT NULL,
template_version TEXT DEFAULT '1.0',
selection_criteria TEXT NOT NULL, -- JSON: complex criteria for model selection
routing_preferences TEXT, -- JSON: preferred routing variants per stage
cost_controls TEXT, -- JSON: budget limits and optimization rules
performance_targets TEXT, -- JSON: latency, quality targets
scope_applicability TEXT NOT NULL, -- JSON: ['minimal', 'basic', 'production']
auto_update BOOLEAN DEFAULT 1, -- Whether to auto-update with new rankings
learning_enabled BOOLEAN DEFAULT 1, -- Whether to learn from usage patterns
expert_mode BOOLEAN DEFAULT 0, -- Whether this requires expert knowledge
created_by TEXT, -- User ID or 'system'
usage_count INTEGER DEFAULT 0,
effectiveness_score REAL, -- Measured effectiveness over time
last_updated TEXT DEFAULT CURRENT_TIMESTAMP,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Model selection history and learning
CREATE TABLE IF NOT EXISTS model_selection_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL,
selection_criteria TEXT NOT NULL, -- JSON: criteria used for selection
selected_models TEXT NOT NULL, -- JSON: {generator: 'model_id', refiner: 'model_id', etc.}
selection_reasoning TEXT, -- JSON: why each model was selected
alternative_models TEXT, -- JSON: other models that were considered
selection_performance TEXT, -- JSON: how well the selection performed
user_feedback INTEGER, -- 1-5 rating of selection quality
learning_applied BOOLEAN DEFAULT 0, -- Whether selection influenced future choices
template_used TEXT, -- Template ID if selection was template-based
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Performance metrics for analytics and monitoring
CREATE TABLE IF NOT EXISTS performance_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL,
conversation_id TEXT,
total_duration INTEGER NOT NULL, -- Total pipeline duration in ms
overall_efficiency REAL DEFAULT 0, -- Overall efficiency score
openrouter_latency INTEGER DEFAULT 0, -- OpenRouter API latency
memory_usage INTEGER DEFAULT 0, -- Memory usage in bytes
total_cost REAL DEFAULT 0, -- Total cost in USD
quality_score REAL DEFAULT 0, -- Quality score 0-10
metrics_data TEXT, -- JSON: Full PerformanceMetrics object
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id)
);
-- Real-time activity tracking for dashboards
CREATE TABLE IF NOT EXISTS activity_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event_type TEXT NOT NULL CHECK (event_type IN (
'query_start', 'query_complete', 'query_error',
'model_switch', 'fallback', 'rate_limit',
'stage_start', 'stage_complete', 'stage_error',
'pipeline_complete', 'export', 'budget_alert',
'cost_alert', 'quality_alert', 'performance_alert'
)),
conversation_id TEXT,
user_id TEXT,
stage TEXT CHECK (stage IN ('generator', 'refiner', 'validator', 'curator')),
model_used TEXT,
provider_name TEXT,
cost REAL,
duration_ms INTEGER,
tokens_used INTEGER,
error_message TEXT,
metadata TEXT, -- JSON for additional event-specific data
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (conversation_id) REFERENCES conversations(id),
FOREIGN KEY (user_id) REFERENCES users(id)
);
`);
// Create performance indexes
await db.exec(`
-- Foreign key indexes
CREATE INDEX IF NOT EXISTS idx_configurations_user ON configurations(user_id);
CREATE INDEX IF NOT EXISTS idx_models_provider ON openrouter_models(provider_id);
CREATE INDEX IF NOT EXISTS idx_pipeline_profiles_user ON pipeline_profiles(user_id);
CREATE INDEX IF NOT EXISTS idx_consensus_profiles_pipeline ON consensus_profiles(pipeline_profile_id);
CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id);
CREATE INDEX IF NOT EXISTS idx_conversations_profile ON conversations(consensus_profile_id);
CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id);
CREATE INDEX IF NOT EXISTS idx_usage_user ON usage_records(user_id);
CREATE INDEX IF NOT EXISTS idx_usage_timestamp ON usage_records(timestamp);
CREATE INDEX IF NOT EXISTS idx_budget_user ON budget_limits(user_id);
-- Search indexes
CREATE INDEX IF NOT EXISTS idx_models_name ON openrouter_models(name);
CREATE INDEX IF NOT EXISTS idx_models_provider_name ON openrouter_models(provider_name);
CREATE INDEX IF NOT EXISTS idx_profiles_name ON pipeline_profiles(name);
CREATE INDEX IF NOT EXISTS idx_profiles_default ON pipeline_profiles(is_default);
CREATE INDEX IF NOT EXISTS idx_usage_conversation ON conversation_usage(conversation_id);
CREATE INDEX IF NOT EXISTS idx_usage_timestamp_legacy ON conversation_usage(timestamp);
-- Knowledge database indexes
CREATE INDEX IF NOT EXISTS idx_knowledge_conversations_id ON knowledge_conversations(conversation_id);
CREATE INDEX IF NOT EXISTS idx_knowledge_conversations_created ON knowledge_conversations(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_knowledge_conversations_question ON knowledge_conversations(question);
CREATE INDEX IF NOT EXISTS idx_curator_truths_conv ON curator_truths(conversation_id);
CREATE INDEX IF NOT EXISTS idx_curator_truths_confidence ON curator_truths(confidence_score DESC);
CREATE INDEX IF NOT EXISTS idx_conversation_context_conv ON conversation_context(conversation_id);
CREATE INDEX IF NOT EXISTS idx_conversation_context_ref ON conversation_context(referenced_conversation_id);
CREATE INDEX IF NOT EXISTS idx_conversation_context_type ON conversation_context(context_type);
CREATE INDEX IF NOT EXISTS idx_conversation_topics_conv ON conversation_topics(conversation_id);
CREATE INDEX IF NOT EXISTS idx_conversation_keywords_conv ON conversation_keywords(conversation_id);
-- Improvement patterns indexes
CREATE INDEX IF NOT EXISTS idx_improvement_patterns_conv ON improvement_patterns(conversation_id);
CREATE INDEX IF NOT EXISTS idx_improvement_patterns_type ON improvement_patterns(improvement_type);
CREATE INDEX IF NOT EXISTS idx_improvement_patterns_question_type ON improvement_patterns(question_type);
CREATE INDEX IF NOT EXISTS idx_improvement_patterns_stages ON improvement_patterns(from_stage, to_stage);
-- Pending sync indexes
CREATE INDEX IF NOT EXISTS idx_conversation_threads_child ON conversation_threads(child_conversation_id);
CREATE INDEX IF NOT EXISTS idx_conversation_threads_parent ON conversation_threads(parent_conversation_id);
CREATE INDEX IF NOT EXISTS idx_conversation_threads_type ON conversation_threads(thread_type);
CREATE INDEX IF NOT EXISTS idx_conversation_threads_created ON conversation_threads(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_pending_sync_user ON pending_sync(user_id);
CREATE INDEX IF NOT EXISTS idx_pending_sync_created ON pending_sync(created_at);
CREATE INDEX IF NOT EXISTS idx_pending_sync_attempts ON pending_sync(sync_attempts);
-- ===== DYNAMIC CONSENSUS & OPENROUTER INTELLIGENCE INDEXES =====
-- Provider performance indexes
CREATE INDEX IF NOT EXISTS idx_provider_performance_model ON provider_performance(model_internal_id);
CREATE INDEX IF NOT EXISTS idx_provider_performance_provider ON provider_performance(provider_name);
CREATE INDEX IF NOT EXISTS idx_provider_performance_variant ON provider_performance(routing_variant);
CREATE INDEX IF NOT EXISTS idx_provider_performance_measured ON provider_performance(measured_at DESC);
CREATE INDEX IF NOT EXISTS idx_provider_performance_success ON provider_performance(success_rate DESC);
CREATE INDEX IF NOT EXISTS idx_provider_performance_latency ON provider_performance(avg_latency_ms ASC);
-- Model rankings indexes
CREATE INDEX IF NOT EXISTS idx_model_rankings_model ON model_rankings(model_internal_id);
CREATE INDEX IF NOT EXISTS idx_model_rankings_source ON model_rankings(ranking_source);
CREATE INDEX IF NOT EXISTS idx_model_rankings_position ON model_rankings(rank_position ASC);
CREATE INDEX IF NOT EXISTS idx_model_rankings_collected ON model_rankings(collected_at DESC);
CREATE INDEX IF NOT EXISTS idx_model_rankings_period ON model_rankings(period_start, period_end);
CREATE INDEX IF NOT EXISTS idx_model_rankings_score ON model_rankings(relative_score DESC);
-- Consensus metrics indexes
CREATE INDEX IF NOT EXISTS idx_consensus_metrics_conv ON consensus_metrics(conversation_id);
CREATE INDEX IF NOT EXISTS idx_consensus_metrics_improvement ON consensus_metrics(improvement_score DESC);
CREATE INDEX IF NOT EXISTS idx_consensus_metrics_complexity ON consensus_metrics(question_complexity);
CREATE INDEX IF NOT EXISTS idx_consensus_metrics_category ON consensus_metrics(question_category);
CREATE INDEX IF NOT EXISTS idx_consensus_metrics_rating ON consensus_metrics(user_rating DESC);
CREATE INDEX IF NOT EXISTS idx_consensus_metrics_created ON consensus_metrics(created_at DESC);
-- Cost analytics indexes
CREATE INDEX IF NOT EXISTS idx_cost_analytics_conv ON cost_analytics(conversation_id);
CREATE INDEX IF NOT EXISTS idx_cost_analytics_cost ON cost_analytics(total_cost DESC);
CREATE INDEX IF NOT EXISTS idx_cost_analytics_efficiency ON cost_analytics(cost_efficiency_score DESC);
CREATE INDEX IF NOT EXISTS idx_cost_analytics_created ON cost_analytics(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_cost_analytics_budget ON cost_analytics(budget_utilized_percentage DESC);
-- Feature usage indexes
CREATE INDEX IF NOT EXISTS idx_feature_usage_conv ON feature_usage(conversation_id);
CREATE INDEX IF NOT EXISTS idx_feature_usage_created ON feature_usage(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_feature_usage_fallbacks ON feature_usage(fallback_events DESC);
CREATE INDEX IF NOT EXISTS idx_feature_usage_switches ON feature_usage(provider_switches DESC);
-- Profile templates indexes
CREATE INDEX IF NOT EXISTS idx_profile_templates_name ON profile_templates(name);
CREATE INDEX IF NOT EXISTS idx_profile_templates_auto_update ON profile_templates(auto_update);
CREATE INDEX IF NOT EXISTS idx_profile_templates_effectiveness ON profile_templates(effectiveness_score DESC);
CREATE INDEX IF NOT EXISTS idx_profile_templates_usage ON profile_templates(usage_count DESC);
CREATE INDEX IF NOT EXISTS idx_profile_templates_expert ON profile_templates(expert_mode);
-- Model selection history indexes
CREATE INDEX IF NOT EXISTS idx_model_selection_conv ON model_selection_history(conversation_id);
CREATE INDEX IF NOT EXISTS idx_model_selection_template ON model_selection_history(template_used);
CREATE INDEX IF NOT EXISTS idx_model_selection_feedback ON model_selection_history(user_feedback DESC);
CREATE INDEX IF NOT EXISTS idx_model_selection_created ON model_selection_history(created_at DESC);
-- Performance metrics indexes
CREATE INDEX IF NOT EXISTS idx_performance_metrics_conv ON performance_metrics(conversation_id);
CREATE INDEX IF NOT EXISTS idx_performance_metrics_timestamp ON performance_metrics(timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_performance_metrics_duration ON performance_metrics(total_duration);
CREATE INDEX IF NOT EXISTS idx_performance_metrics_cost ON performance_metrics(total_cost);
-- Activity log indexes for efficient dashboard queries
CREATE INDEX IF NOT EXISTS idx_activity_log_event ON activity_log(event_type);
CREATE INDEX IF NOT EXISTS idx_activity_log_created ON activity_log(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_activity_log_conversation ON activity_log(conversation_id);
CREATE INDEX IF NOT EXISTS idx_activity_log_user ON activity_log(user_id);
CREATE INDEX IF NOT EXISTS idx_activity_log_stage ON activity_log(stage);
`);
// Run migration to handle existing data with bulletproof model references
await runModelNormalizationMigration();
// Run migration to add user tracking to conversation_usage
await runConversationUsageMigration();
// Run migration to remove foreign keys from pending_sync (for better UX)
await runPendingSyncMigration();
// Run migration to add analytics columns to conversations table
await runConversationAnalyticsMigration();
// Mark as initialized (seeding moved to setup-claude-code)
isInitialized = true;
console.error('✅ Unified database initialized successfully');
return true;
}
catch (error) {
console.error('❌ Unified database initialization error:', error);
return false;
}
}
/**
* BULLETPROOF MODEL MANAGEMENT FUNCTIONS
* These functions provide stable model references that survive OpenRouter model ID changes
*/
/**
* Migration function to normalize existing model data
*/
async function runModelNormalizationMigration() {
try {
// Check if migration has already been run AND if sync_metadata has all required columns
const migrationCheck = await db.get("SELECT value FROM settings WHERE key = 'model_normalization_migrated'");
// Also check if sync_metadata actually has the required columns
let needsSyncMetadataMigration = false;
try {
const syncTableExists = await db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='sync_metadata'");
if (syncTableExists) {
const columns = await db.all("PRAGMA table_info(sync_metadata)");
const columnNames = columns.map(col => col.name);
needsSyncMetadataMigration = !columnNames.includes('rankings_last_synced') ||
!columnNames.includes('provider_performance_synced') ||
!columnNames.includes('intelligence_version');
}
}
catch (checkError) {
needsSyncMetadataMigration = true; // Default to migration on error
}
if (migrationCheck?.value === 'true' && !needsSyncMetadataMigration) {
return; // Migration already completed and columns exist
}
console.log('🔄 Running model normalization migration...');
// Step 1: Handle schema evolution - add new columns and update models table structure
try {
// First, check current schema and add missing columns
await db.exec(`
-- Add new columns to pipeline_profiles if they don't exist
ALTER TABLE pipeline_profiles ADD COLUMN generator_model_internal_id INTEGER;
`);
}
catch (error) {
// Column may already exist, that's okay
}
try {
await db.exec(`
ALTER TABLE pipeline_profiles ADD COLUMN refiner_model_internal_id INTEGER;
ALTER TABLE pipeline_profiles ADD COLUMN validator_model_internal_id INTEGER;
ALTER TABLE pipeline_profiles ADD COLUMN curator_model_internal_id INTEGER;
`);
}
catch (error) {
// Columns may already exist, that's okay
}
// Step 1.4: Add dynamic intelligence columns to existing tables
try {
await db.exec(`
-- Add dynamic template columns to pipeline_profiles
ALTER TABLE pipeline_profiles ADD COLUMN is_dynamic_template BOOLEAN DEFAULT 0;
ALTER TABLE pipeline_profiles ADD COLUMN selection_criteria TEXT;
ALTER TABLE pipeline_profiles ADD COLUMN routing_preferences TEXT;
ALTER TABLE pipeline_profiles ADD COLUMN cost_controls TEXT;
`);
}
catch (error) {
// Columns may already exist, that's okay
}
try {
await db.exec(`
-- Enhance improvement_patterns table
ALTER TABLE improvement_patterns ADD COLUMN before_content TEXT;
ALTER TABLE improvement_patterns ADD COLUMN after_content TEXT;
ALTER TABLE improvement_patterns ADD COLUMN improvement_score REAL;
ALTER TABLE improvement_patterns ADD COLUMN meta_commentary TEXT;
`);
}
catch (error) {
// Columns may already exist, that's okay
}
try {
// Check if sync_metadata table exists first
const syncTableExists = await db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='sync_metadata'");
if (syncTableExists) {
// Check which columns exist
const columns = await db.all("PRAGMA table_info(sync_metadata)");
const columnNames = columns.map(col => col.name);
// Check if we need to recreate the table (if too many columns are missing)
const missingColumns = ['rankings_last_synced', 'provider_performance_synced', 'intelligence_version']
.filter(col => !columnNames.includes(col));
if (missingColumns.length >= 2) {
console.log('🔄 Recreating sync_metadata table with correct schema...');
try {
// Save existing data
const existingData = await db.all('SELECT * FROM sync_metadata');
// Drop and recreate table with correct schema
await db.exec('DROP TABLE sync_metadata');
await db.exec(`
CREATE TABLE sync_metadata (
id TEXT PRIMARY KEY,
sync_type TEXT NOT NULL,
started_at TEXT NOT NULL,
completed_at TEXT,
status TEXT NOT NULL,
providers_synced INTEGER DEFAULT 0,
models_synced INTEGER DEFAULT 0,
error_message TEXT,
next_sync_due TEXT NOT NULL,
rankings_last_synced TEXT,
provider_performance_synced TEXT,
intelligence_version TEXT DEFAULT '1.7.0',
created_at TEXT DEFAULT CURRENT_TIMESTAMP
)
`);
// Restore compatible data
for (const row of existingData) {
await db.run(`
INSERT OR IGNORE INTO sync_metadata
(id, sync_type, started_at, completed_at, status, providers_synced, models_synced, error_message, next_sync_due, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, [
row.id, row.sync_type, row.started_at, row.completed_at, row.status,
row.providers_synced || 0, row.models_synced || 0, row.error_message,
row.next_sync_due, row.created_at || new Date().toISOString()
]);
}
console.log('✅ sync_metadata table recreated with all required columns');
}
catch (recreateError) {
console.warn('⚠️ Failed to recreate sync_metadata table:', recreateError);
// Fall back to individual column additions
}
}
else {
// Add missing columns one by one (with individual error handling)
if (!columnNames.includes('rankings_last_synced')) {
try {
await db.exec('ALTER TABLE sync_metadata ADD COLUMN rankings_last_synced TEXT');
console.log('✅ Added rankings_last_synced column to sync_metadata');
}
catch (colError) {
console.warn('⚠️ Failed to add rankings_last_synced column:', colError);
}
}
if (!columnNames.includes('provider_performance_synced')) {
try {
await db.exec('ALTER TABLE sync_metadata ADD COLUMN provider_performance_synced TEXT');
console.log('✅ Added provider_performance_synced column to sync_metadata');
}
catch (colError) {
console.warn('⚠️ Failed to add provider_performance_synced column:', colError);
}
}
if (!columnNames.includes('intelligence_version')) {
try {
await db.exec('ALTER TABLE sync_metadata ADD COLUMN intelligence_version TEXT DEFAULT "1.7.0"');
console.log('✅ Added intelligence_version column to sync_metadata');
}
catch (colError) {
console.warn('⚠️ Failed to add intelligence_version column:', colError);
}
}
}
// Ensure there's a row for openrouter_models sync tracking
await db.run(`
INSERT OR IGNORE INTO sync_metadata
(id, sync_type, started_at, status, next_sync_due, created_at)
VALUES (
'sync_openrouter_models',
'openrouter_models',
datetime('now'),
'pending',
datetime('now', '+1 day'),
datetime('now')
)
`);
console.log('✅ sync_metadata migration completed');
}
}
catch (error) {
console.warn('⚠️ Could not update sync_metadata columns:', error);
// Continue with initialization even if this fails
}
// Step 1.5: Handle openrouter_models table schema evolution
try {
// Check if openrouter_models has the new schema
const columns = await db.all("PRAGMA table_info(openrouter_models)");
const hasInternalId = columns.some(col => col.name === 'internal_id');
const hasOpenrouterId = columns.some(col => col.name === 'openrouter_id');
if (!hasInternalId || !hasOpenrouterId) {
console.log('🔧 Updating openrouter_models table schema...');
// Create new table with correct schema
await db.exec(`
CREATE TABLE openrouter_models_new (
internal_id INTEGER PRIMARY KEY AUTOINCREMENT,
openrouter_id TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
provider_id TEXT NOT NULL,
provider_name TEXT NOT NULL,
description TEXT DEFAULT '',
capabilities TEXT DEFAULT '[]',
pricing_input REAL DEFAULT 0.0,
pricing_output REAL DEFAULT 0.0,
pricing_image REAL,
pricing_request REAL,
context_window INTEGER DEFAULT 4096,
created_at INTEGER NOT NULL,
input_modalities TEXT DEFAULT '["text"]',
output_modalities TEXT DEFAULT '["text"]',
is_active BOOLEAN DEFAULT 1,
last_updated TEXT NOT NULL,
FOREIGN KEY (provider_id) REFERENCES openrouter_providers(id)
);
`);
// Migrate existing data if any exists
try {
const existingModels = await db.all('SELECT * FROM openrouter_models');
for (const model of existingModels) {
await db.run(`
INSERT OR IGNORE INTO openrouter_models_new
(openrouter_id, name, provider_id, provider_name, description, capabilities,
pricing_input, pricing_output, pricing_image, pricing_request, context_window,
created_at, input_modalities, output_modalities, is_active, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, [
model.id || model.openrouter_id,
model.name,
model.provider_id,
model.provider_name,
model.description || '',
model.capabilities || '[]',
model.pricing_input || 0,
model.pricing_output || 0,
model.pricing_image,
model.pricing_request,
model.context_window || 4096,
model.created_at || Date.now(),
model.input_modalities || '["text"]',
model.output_modalities || '["text"]',
model.is_active !== false ? 1 : 0,
model.last_updated || new Date().toISOString()
]);
}
}
catch (migrationError) {
console.log('📝 No existing models to migrate (fresh start)');
}
// Replace old table
await db.exec('DROP TABLE openrouter_models');
await db.exec('ALTER TABLE openrouter_models_new RENAME TO openrouter_models');
console.log('✅ openrouter_models table schema updated');
}
}
catch (error) {
console.error('Schema evolution error:', error);
}
// Step 1.6: Handle activity_log CHECK constraint migration
try {
// Check if activity_log has the correct CHECK constraint with all required event types
const tableInfo = await db.get("SELECT sql FROM sqlite_master WHERE type='table' AND name='activity_log'");
if (tableInfo && tableInfo.sql) {
const currentConstraint = tableInfo.sql;
const hasPipelineComplete = currentConstraint.includes("'pipeline_complete'");
const hasCostAlert = currentConstraint.includes("'cost_alert'");
const hasQualityAlert = currentConstraint.includes("'quality_alert'");
const hasPerformanceAlert = currentConstraint.includes("'performance_alert'");
if (!hasPipelineComplete || !hasCostAlert || !hasQualityAlert || !hasPerformanceAlert) {
console.log('🔧 Updating activity_log table with correct CHECK constraint...');
// Save existing data
const existingData = await db.all('SELECT * FROM activity_log');
// Drop and recreate table with correct CHECK constraint
await db.exec('DROP TABLE activity_log');
await db.