@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
397 lines ā¢ 19.9 kB
JavaScript
/**
* Database Migration Tool
*
* Migrates data from 6 separate SQLite databases into the unified normalized database.
* Handles data consolidation, deduplication, and integrity validation.
*/
import { open } from 'sqlite';
import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs';
import { v4 as uuidv4 } from 'uuid';
import { initializeUnifiedDatabase, getDatabase } from './unified-database.js';
const DB_DIR = path.join(os.homedir(), '.hive-ai');
// Paths to existing databases
const OLD_DATABASES = {
context: path.join(DB_DIR, 'hive-ai-context.db'),
pipeline: path.join(DB_DIR, 'secure-pipeline.db'),
keys: path.join(DB_DIR, 'secure-keys.db'),
openrouter: path.join(DB_DIR, 'openrouter-data.db'),
knowledge: path.join(DB_DIR, 'hive-ai-knowledge.db'),
users: path.join(DB_DIR, 'hive-ai-users.db')
};
// Paths to JSON files
const JSON_FILES = {
usage: path.join(DB_DIR, 'usage-monitor.json'),
budget: path.join(DB_DIR, 'budget.json'),
costTracking: path.join(DB_DIR, 'cost-tracking.json')
};
/**
* Main migration function - consolidates all data sources
*/
export async function migrateAllDatabases() {
console.log('š Starting database consolidation migration...');
const report = {
success: false,
migratedTables: [],
errors: [],
statistics: {
users: 0,
configurations: 0,
providers: 0,
models: 0,
pipelineProfiles: 0,
consensusProfiles: 0,
conversations: 0,
messages: 0,
usageRecords: 0
}
};
try {
// Initialize unified database
console.log('š Initializing unified database...');
await initializeUnifiedDatabase();
const unifiedDb = await getDatabase();
// Start transaction for atomic migration
await unifiedDb.run('BEGIN TRANSACTION');
try {
// Phase 1: Migrate from hive-ai-context.db
if (fs.existsSync(OLD_DATABASES.context)) {
console.log('š„ Migrating from hive-ai-context.db...');
await migrateContextDatabase(unifiedDb, report);
report.migratedTables.push('hive-ai-context.db');
}
// Phase 2: Migrate from secure-pipeline.db
if (fs.existsSync(OLD_DATABASES.pipeline)) {
console.log('š„ Migrating from secure-pipeline.db...');
await migratePipelineDatabase(unifiedDb, report);
report.migratedTables.push('secure-pipeline.db');
}
// Phase 3: Migrate from secure-keys.db
if (fs.existsSync(OLD_DATABASES.keys)) {
console.log('š„ Migrating from secure-keys.db...');
await migrateKeysDatabase(unifiedDb, report);
report.migratedTables.push('secure-keys.db');
}
// Phase 4: Migrate from openrouter-data.db
if (fs.existsSync(OLD_DATABASES.openrouter)) {
console.log('š„ Migrating from openrouter-data.db...');
await migrateOpenRouterDatabase(unifiedDb, report);
report.migratedTables.push('openrouter-data.db');
}
// Phase 5: Migrate from hive-ai-knowledge.db
if (fs.existsSync(OLD_DATABASES.knowledge)) {
console.log('š„ Migrating from hive-ai-knowledge.db...');
await migrateKnowledgeDatabase(unifiedDb, report);
report.migratedTables.push('hive-ai-knowledge.db');
}
// Phase 6: Migrate JSON files
console.log('š„ Migrating JSON files...');
await migrateJsonFiles(unifiedDb, report);
report.migratedTables.push('JSON files');
// Commit transaction
await unifiedDb.run('COMMIT');
report.success = true;
console.log('ā
Database migration completed successfully!');
}
catch (error) {
await unifiedDb.run('ROLLBACK');
throw error;
}
}
catch (error) {
report.errors.push(`Migration failed: ${error.message}`);
console.error('ā Migration failed:', error);
}
return report;
}
/**
* Migrate data from hive-ai-context.db
*/
async function migrateContextDatabase(unifiedDb, report) {
const sqlite3Driver = await import('sqlite3');
const contextDb = await open({
filename: OLD_DATABASES.context,
driver: sqlite3Driver.default.Database,
mode: sqlite3Driver.default.OPEN_READONLY
});
try {
// Migrate configurations
const configs = await contextDb.all('SELECT * FROM configurations');
for (const config of configs) {
await unifiedDb.run('INSERT OR IGNORE INTO configurations (key, value, encrypted, created_at, updated_at) VALUES (?, ?, ?, ?, ?)', config.key, config.value, config.encrypted, config.created_at, config.updated_at);
report.statistics.configurations++;
}
// Migrate user profiles
const userProfiles = await contextDb.all('SELECT * FROM user_profiles');
for (const profile of userProfiles) {
await unifiedDb.run('INSERT OR IGNORE INTO users (id, email, license_key, tier, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)', profile.id, profile.email, profile.license_key, profile.tier, profile.created_at, profile.updated_at);
report.statistics.users++;
}
// Migrate consensus profiles
const consensusProfiles = await contextDb.all('SELECT * FROM consensus_profiles');
for (const profile of consensusProfiles) {
await unifiedDb.run('INSERT OR IGNORE INTO consensus_profiles (id, profile_name, generator_model, refiner_model, validator_model, curator_model, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', profile.id, profile.profile_name, profile.generator_model, profile.refiner_model, profile.validator_model, profile.curator_model, profile.created_at, profile.updated_at);
report.statistics.consensusProfiles++;
}
// Migrate consensus settings
const consensusSettings = await contextDb.all('SELECT * FROM consensus_settings');
for (const setting of consensusSettings) {
await unifiedDb.run('INSERT OR IGNORE INTO consensus_settings (key, value, updated_at) VALUES (?, ?, ?)', setting.key, setting.value, setting.updated_at);
}
// Migrate conversations
const conversations = await contextDb.all('SELECT * FROM conversations');
for (const conv of conversations) {
await unifiedDb.run('INSERT OR IGNORE INTO conversations (id, created_at, updated_at) VALUES (?, ?, ?)', conv.id, conv.created_at, conv.last_updated || conv.updated_at);
report.statistics.conversations++;
}
// Migrate messages
const messages = await contextDb.all('SELECT * FROM messages');
for (const msg of messages) {
await unifiedDb.run('INSERT OR IGNORE INTO messages (id, conversation_id, role, content, timestamp) VALUES (?, ?, ?, ?, ?)', msg.id, msg.conversation_id, msg.role, msg.content, msg.timestamp);
report.statistics.messages++;
}
// Migrate conversation usage
const usage = await contextDb.all('SELECT * FROM conversation_usage');
for (const record of usage) {
await unifiedDb.run('INSERT OR IGNORE INTO conversation_usage (id, conversation_id, timestamp) VALUES (?, ?, ?)', record.id, record.conversation_id, record.timestamp);
}
}
finally {
await contextDb.close();
}
}
/**
* Migrate data from secure-pipeline.db (wizard profiles)
*/
async function migratePipelineDatabase(unifiedDb, report) {
const sqlite3Driver = await import('sqlite3');
const pipelineDb = await open({
filename: OLD_DATABASES.pipeline,
driver: sqlite3Driver.default.Database,
mode: sqlite3Driver.default.OPEN_READONLY
});
try {
// Get all profiles from secure pipeline
const profiles = await pipelineDb.all('SELECT * FROM pipeline_profiles');
for (const profile of profiles) {
// Parse individual stage data (stored as JSON strings)
const generatorData = JSON.parse(profile.generator_data);
const refinerData = JSON.parse(profile.refiner_data);
const validatorData = JSON.parse(profile.validator_data);
const curatorData = profile.curator_data ? JSON.parse(profile.curator_data) : validatorData;
const pipelineProfileId = uuidv4();
// Insert into pipeline_profiles table
await unifiedDb.run(`INSERT OR IGNORE INTO pipeline_profiles
(id, name, is_default, generator_provider, generator_model, generator_temperature,
refiner_provider, refiner_model, refiner_temperature,
validator_provider, validator_model, validator_temperature,
curator_provider, curator_model, curator_temperature, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, pipelineProfileId, profile.name, profile.is_default ? 1 : 0, generatorData.provider_name || 'openrouter', generatorData.model || 'openai/gpt-4', generatorData.temperature || 0.7, refinerData.provider_name || 'openrouter', refinerData.model || 'anthropic/claude-3-sonnet', refinerData.temperature || 0.7, validatorData.provider_name || 'openrouter', validatorData.model || 'google/gemini-pro', validatorData.temperature || 0.7, curatorData.provider_name || 'openrouter', curatorData.model || 'meta-llama/llama-2-70b-chat', curatorData.temperature || 0.7, profile.created_at || new Date().toISOString(), profile.updated_at || new Date().toISOString());
report.statistics.pipelineProfiles++;
}
}
finally {
await pipelineDb.close();
}
}
/**
* Migrate data from secure-keys.db (API keys)
*/
async function migrateKeysDatabase(unifiedDb, report) {
const sqlite3Driver = await import('sqlite3');
const keysDb = await open({
filename: OLD_DATABASES.keys,
driver: sqlite3Driver.default.Database,
mode: sqlite3Driver.default.OPEN_READONLY
});
try {
// Get all API keys from providers table
const providers = await keysDb.all('SELECT * FROM providers');
for (const provider of providers) {
// Store API keys as encrypted configurations
await unifiedDb.run('INSERT OR IGNORE INTO configurations (key, value, encrypted, created_at, updated_at) VALUES (?, ?, 1, ?, ?)', `${provider.name}_api_key`, provider.api_key_encrypted, provider.created_at || new Date().toISOString(), provider.updated_at || new Date().toISOString());
report.statistics.configurations++;
}
}
finally {
await keysDb.close();
}
}
/**
* Migrate data from openrouter-data.db
*/
async function migrateOpenRouterDatabase(unifiedDb, report) {
const sqlite3Driver = await import('sqlite3');
const openrouterDb = await open({
filename: OLD_DATABASES.openrouter,
driver: sqlite3Driver.default.Database,
mode: sqlite3Driver.default.OPEN_READONLY
});
try {
// Migrate providers
const providers = await openrouterDb.all('SELECT * FROM openrouter_providers');
for (const provider of providers) {
await unifiedDb.run(`INSERT OR IGNORE INTO openrouter_providers
(id, name, display_name, model_count, average_cost, capabilities, is_active, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, provider.id, provider.name, provider.display_name, provider.model_count, provider.average_cost, provider.capabilities, provider.is_active, provider.created_at, provider.last_updated);
report.statistics.providers++;
}
// Migrate models
const models = await openrouterDb.all('SELECT * FROM openrouter_models');
for (const model of models) {
await unifiedDb.run(`INSERT OR IGNORE INTO openrouter_models
(id, name, provider_id, provider_name, description, capabilities,
pricing_input, pricing_output, pricing_image, pricing_request,
context_window, created_timestamp, input_modalities, output_modalities,
is_active, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, model.id, model.name, model.provider_id, model.provider_name, model.description, model.capabilities, model.pricing_input, model.pricing_output, model.pricing_image, model.pricing_request, model.context_window, model.created_at, model.input_modalities, model.output_modalities, model.is_active, model.created_at, model.last_updated);
report.statistics.models++;
}
// Migrate sync metadata
const syncData = await openrouterDb.all('SELECT * FROM openrouter_sync_metadata');
for (const sync of syncData) {
await unifiedDb.run(`INSERT OR IGNORE INTO sync_metadata
(id, sync_type, started_at, completed_at, status, providers_synced,
models_synced, error_message, next_sync_due, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, sync.id, sync.sync_type, sync.started_at, sync.completed_at, sync.status, sync.providers_synced, sync.models_synced, sync.error_message, sync.next_sync_due, sync.created_at);
}
}
finally {
await openrouterDb.close();
}
}
/**
* Migrate data from hive-ai-knowledge.db
*/
async function migrateKnowledgeDatabase(unifiedDb, report) {
const sqlite3Driver = await import('sqlite3');
const knowledgeDb = await open({
filename: OLD_DATABASES.knowledge,
driver: sqlite3Driver.default.Database,
mode: sqlite3Driver.default.OPEN_READONLY
});
try {
// Migrate knowledge conversations
const knowledgeConversations = await knowledgeDb.all('SELECT * FROM conversations');
for (const conv of knowledgeConversations) {
await unifiedDb.run(`INSERT OR IGNORE INTO knowledge_conversations
(id, conversation_id, question, final_answer, source_of_truth, conversation_context, profile_id, created_at, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, conv.id, conv.id, conv.question, conv.final_answer, conv.source_of_truth, conv.conversation_context, conv.profile_id, conv.created_at, conv.last_updated);
report.statistics.conversations++;
}
// Migrate curator truths if they exist
try {
const curatorTruths = await knowledgeDb.all('SELECT * FROM curator_truths');
for (const truth of curatorTruths) {
await unifiedDb.run(`INSERT OR IGNORE INTO curator_truths
(conversation_id, curator_output, confidence_score, topic_summary, created_at)
VALUES (?, ?, ?, ?, ?)`, truth.conversation_id, truth.curator_output, truth.confidence_score, truth.topic_summary, truth.created_at);
}
}
catch (error) {
// curator_truths table might not exist or have data
console.log('No curator_truths data to migrate');
}
// Migrate conversation context if it exists
try {
const contextData = await knowledgeDb.all('SELECT * FROM conversation_context');
for (const context of contextData) {
await unifiedDb.run(`INSERT OR IGNORE INTO conversation_context
(conversation_id, referenced_conversation_id, relevance_score, context_type, created_at)
VALUES (?, ?, ?, ?, ?)`, context.conversation_id, context.referenced_conversation_id, context.relevance_score, context.context_type, context.created_at);
}
}
catch (error) {
// conversation_context table might not exist or have data
console.log('No conversation_context data to migrate');
}
}
finally {
await knowledgeDb.close();
}
}
/**
* Migrate data from JSON files
*/
async function migrateJsonFiles(unifiedDb, report) {
// Migrate usage monitoring data
if (fs.existsSync(JSON_FILES.usage)) {
try {
const usageData = JSON.parse(fs.readFileSync(JSON_FILES.usage, 'utf8'));
for (const [conversationId, data] of Object.entries(usageData)) {
const record = data;
await unifiedDb.run(`INSERT OR IGNORE INTO usage_records
(id, conversation_id, action_type, cost, tokens_input, tokens_output, model_used, timestamp)
VALUES (?, ?, 'conversation', ?, ?, ?, ?, ?)`, uuidv4(), conversationId, record.cost || 0, record.tokens_input || 0, record.tokens_output || 0, record.model_used || null, record.timestamp || new Date().toISOString());
report.statistics.usageRecords++;
}
}
catch (error) {
report.errors.push(`Failed to migrate usage-monitor.json: ${error}`);
}
}
// Migrate budget data
if (fs.existsSync(JSON_FILES.budget)) {
try {
const budgetData = JSON.parse(fs.readFileSync(JSON_FILES.budget, 'utf8'));
await unifiedDb.run(`INSERT OR IGNORE INTO budget_limits
(id, user_id, period_type, limit_amount, current_spent, period_start, period_end, is_active)
VALUES (?, 'default', 'daily', ?, ?, ?, ?, 1)`, uuidv4(), budgetData.daily_limit || 100, budgetData.current_spent || 0, budgetData.period_start || new Date().toISOString(), budgetData.period_end || new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString());
}
catch (error) {
report.errors.push(`Failed to migrate budget.json: ${error}`);
}
}
}
/**
* Create backup of old databases before migration
*/
export async function createBackup() {
const backupDir = path.join(DB_DIR, 'backup-' + Date.now());
fs.mkdirSync(backupDir, { recursive: true });
for (const [name, dbPath] of Object.entries(OLD_DATABASES)) {
if (fs.existsSync(dbPath)) {
const backupPath = path.join(backupDir, path.basename(dbPath));
fs.copyFileSync(dbPath, backupPath);
console.log(`š Backed up ${name} database`);
}
}
for (const [name, jsonPath] of Object.entries(JSON_FILES)) {
if (fs.existsSync(jsonPath)) {
const backupPath = path.join(backupDir, path.basename(jsonPath));
fs.copyFileSync(jsonPath, backupPath);
console.log(`š Backed up ${name} JSON file`);
}
}
console.log(`ā
Backup created at: ${backupDir}`);
}
/**
* Print migration report
*/
export function printMigrationReport(report) {
console.log('\nš Migration Report:');
console.log('===================');
if (report.success) {
console.log('ā
Status: SUCCESS');
}
else {
console.log('ā Status: FAILED');
}
console.log(`š Migrated sources: ${report.migratedTables.join(', ')}`);
console.log('\nš Statistics:');
console.log(` Users: ${report.statistics.users}`);
console.log(` Configurations: ${report.statistics.configurations}`);
console.log(` Providers: ${report.statistics.providers}`);
console.log(` Models: ${report.statistics.models}`);
console.log(` Pipeline Profiles: ${report.statistics.pipelineProfiles}`);
console.log(` Consensus Profiles: ${report.statistics.consensusProfiles}`);
console.log(` Conversations: ${report.statistics.conversations}`);
console.log(` Messages: ${report.statistics.messages}`);
console.log(` Usage Records: ${report.statistics.usageRecords}`);
if (report.errors.length > 0) {
console.log('\nā ļø Errors:');
report.errors.forEach(error => console.log(` ā¢ ${error}`));
}
}
//# sourceMappingURL=database-migration.js.map