claude-flow-multilang
Version:
Revolutionary multilingual AI orchestration framework with cultural awareness and DDD architecture
873 lines (771 loc) • 23.9 kB
JavaScript
/**
* Hive Mind Database Optimizer
*
* Safe, backward-compatible database optimization for existing deployments
* Adds indexes, performance improvements, and new features without breaking changes
*/
import Database from 'better-sqlite3';
import { existsSync, mkdirSync } from 'fs';
import { writeFile, readFile } from 'fs/promises';
import path from 'path';
import chalk from 'chalk';
import ora from 'ora';
/**
* Optimize existing hive mind database with backward compatibility
*/
export async function optimizeHiveMindDatabase(dbPath, options = {}) {
const spinner = ora('Optimizing Hive Mind database...').start();
try {
// Open database with write-ahead logging for better performance
const db = new Database(dbPath, {
verbose: options.verbose ? console.log : null,
});
// Enable WAL mode for better concurrency
db.pragma('journal_mode = WAL');
db.pragma('synchronous = NORMAL');
// Get current schema version
const schemaVersion = getSchemaVersion(db);
spinner.text = `Current schema version: ${schemaVersion}`;
// Apply optimizations based on version
let optimizationsApplied = [];
// Version 1.0 -> 1.1: Add basic indexes
if (schemaVersion < 1.1) {
spinner.text = 'Applying performance indexes...';
applyBasicIndexes(db);
optimizationsApplied.push('Basic performance indexes');
}
// Version 1.1 -> 1.2: Add advanced indexes and analyze
if (schemaVersion < 1.2) {
spinner.text = 'Applying advanced indexes...';
applyAdvancedIndexes(db);
optimizationsApplied.push('Advanced query optimization');
}
// Version 1.2 -> 1.3: Add performance tracking tables
if (schemaVersion < 1.3) {
spinner.text = 'Adding performance tracking...';
addPerformanceTracking(db);
optimizationsApplied.push('Performance monitoring tables');
}
// Version 1.3 -> 1.4: Add memory optimization
if (schemaVersion < 1.4) {
spinner.text = 'Optimizing memory management...';
addMemoryOptimization(db);
optimizationsApplied.push('Memory optimization features');
}
// Version 1.4 -> 1.5: Add behavioral tracking
if (schemaVersion < 1.5) {
spinner.text = 'Adding behavioral analysis...';
addBehavioralTracking(db);
optimizationsApplied.push('Behavioral pattern tracking');
}
// Run ANALYZE to update query planner statistics
spinner.text = 'Updating query statistics...';
db.exec('ANALYZE');
// Vacuum if requested (requires exclusive access)
if (options.vacuum) {
spinner.text = 'Vacuuming database...';
db.exec('VACUUM');
optimizationsApplied.push('Database vacuumed');
}
// Update schema version
updateSchemaVersion(db, 1.5);
// Close database
db.close();
spinner.succeed('Database optimization complete!');
if (optimizationsApplied.length > 0) {
console.log('\n' + chalk.green('✓') + ' Optimizations applied:');
optimizationsApplied.forEach((opt) => {
console.log(' - ' + opt);
});
} else {
console.log('\n' + chalk.yellow('ℹ') + ' Database already optimized');
}
return { success: true, optimizations: optimizationsApplied };
} catch (error) {
spinner.fail('Database optimization failed');
console.error(chalk.red('Error:'), error.message);
return { success: false, error: error.message };
}
}
/**
* Get current schema version
*/
function getSchemaVersion(db) {
try {
// Check if schema_version table exists
const tableExists = db
.prepare(
`
SELECT name FROM sqlite_master
WHERE type='table' AND name='schema_version'
`,
)
.get();
if (!tableExists) {
// Create schema version table
db.exec(`
CREATE TABLE schema_version (
version REAL PRIMARY KEY,
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP,
description TEXT
)
`);
// Insert initial version
db.prepare(
`
INSERT INTO schema_version (version, description)
VALUES (1.0, 'Initial schema')
`,
).run();
return 1.0;
}
// Get latest version
const result = db
.prepare(
`
SELECT version FROM schema_version
ORDER BY version DESC LIMIT 1
`,
)
.get();
return result ? result.version : 1.0;
} catch (error) {
// If any error, assume version 1.0
return 1.0;
}
}
/**
* Update schema version
*/
function updateSchemaVersion(db, version, description = '') {
db.prepare(
`
INSERT OR REPLACE INTO schema_version (version, description)
VALUES (?, ?)
`,
).run(version, description || `Updated to version ${version}`);
}
/**
* Apply basic performance indexes
*/
function applyBasicIndexes(db) {
// First ensure all required columns exist
ensureRequiredColumns(db);
// Check which tables exist before creating indexes
const tables = db
.prepare(
`
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
`,
)
.all()
.map((row) => row.name);
const tableSet = new Set(tables);
const indexes = [];
// Only create indexes for tables that exist
if (tableSet.has('swarms')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_swarms_status ON swarms(status)',
'CREATE INDEX IF NOT EXISTS idx_swarms_created ON swarms(created_at)',
);
}
if (tableSet.has('agents')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_agents_swarm ON agents(swarm_id)',
'CREATE INDEX IF NOT EXISTS idx_agents_type ON agents(type)',
'CREATE INDEX IF NOT EXISTS idx_agents_status ON agents(status)',
);
}
if (tableSet.has('tasks')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_tasks_swarm ON tasks(swarm_id)',
'CREATE INDEX IF NOT EXISTS idx_tasks_agent ON tasks(agent_id)',
'CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)',
'CREATE INDEX IF NOT EXISTS idx_tasks_priority ON tasks(priority DESC)',
);
}
if (tableSet.has('collective_memory')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_memory_swarm ON collective_memory(swarm_id)',
'CREATE INDEX IF NOT EXISTS idx_memory_key ON collective_memory(key)',
'CREATE INDEX IF NOT EXISTS idx_memory_type ON collective_memory(type)',
);
}
if (tableSet.has('consensus_decisions')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_consensus_swarm ON consensus_decisions(swarm_id)',
'CREATE INDEX IF NOT EXISTS idx_consensus_created ON consensus_decisions(created_at)',
);
}
indexes.forEach((sql) => {
try {
db.exec(sql);
} catch (error) {
console.warn(`Warning: Could not create index: ${error.message}`);
}
});
}
/**
* Ensure all required columns exist
*/
function ensureRequiredColumns(db) {
// First check which tables exist
const tables = db
.prepare(
`
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
`,
)
.all()
.map((row) => row.name);
const tableSet = new Set(tables);
// Only check columns for tables that exist
if (tableSet.has('tasks')) {
// Check and add priority column to tasks table
const hasPriority = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('tasks')
WHERE name = 'priority'
`,
)
.get();
if (!hasPriority || hasPriority.count === 0) {
try {
db.exec('ALTER TABLE tasks ADD COLUMN priority INTEGER DEFAULT 5');
console.log('Added missing priority column to tasks table');
} catch (error) {
if (
!error.message.includes('duplicate column') &&
!error.message.includes('no such table')
) {
throw error;
}
}
}
// Check and add completed_at column to tasks table
const hasCompletedAt = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('tasks')
WHERE name = 'completed_at'
`,
)
.get();
if (!hasCompletedAt || hasCompletedAt.count === 0) {
try {
db.exec('ALTER TABLE tasks ADD COLUMN completed_at DATETIME');
console.log('Added missing completed_at column to tasks table');
} catch (error) {
if (
!error.message.includes('duplicate column') &&
!error.message.includes('no such table')
) {
throw error;
}
}
}
// Check and add result column to tasks table
const hasResult = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('tasks')
WHERE name = 'result'
`,
)
.get();
if (!hasResult || hasResult.count === 0) {
try {
db.exec('ALTER TABLE tasks ADD COLUMN result TEXT');
console.log('Added missing result column to tasks table');
} catch (error) {
if (
!error.message.includes('duplicate column') &&
!error.message.includes('no such table')
) {
throw error;
}
}
}
}
if (tableSet.has('swarms')) {
// Check and add updated_at column to swarms table
const hasUpdatedAt = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('swarms')
WHERE name = 'updated_at'
`,
)
.get();
if (!hasUpdatedAt || hasUpdatedAt.count === 0) {
try {
db.exec('ALTER TABLE swarms ADD COLUMN updated_at DATETIME');
console.log('Added missing updated_at column to swarms table');
} catch (error) {
if (
!error.message.includes('duplicate column') &&
!error.message.includes('no such table')
) {
throw error;
}
}
}
}
}
/**
* Apply advanced performance indexes
*/
function applyAdvancedIndexes(db) {
// Check which tables exist
const tables = db
.prepare(
`
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
`,
)
.all()
.map((row) => row.name);
const tableSet = new Set(tables);
const indexes = [];
// Composite indexes for common queries
if (tableSet.has('tasks')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_tasks_swarm_status ON tasks(swarm_id, status)',
'CREATE INDEX IF NOT EXISTS idx_tasks_full ON tasks(swarm_id, agent_id, status, priority)',
"CREATE INDEX IF NOT EXISTS idx_tasks_pending ON tasks(swarm_id, priority) WHERE status = 'pending'",
);
}
if (tableSet.has('agents')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_agents_swarm_type ON agents(swarm_id, type)',
'CREATE INDEX IF NOT EXISTS idx_agents_full ON agents(swarm_id, type, status, role)',
);
}
if (tableSet.has('collective_memory')) {
indexes.push(
'CREATE INDEX IF NOT EXISTS idx_memory_swarm_key ON collective_memory(swarm_id, key)',
);
}
if (tableSet.has('swarms')) {
indexes.push(
"CREATE INDEX IF NOT EXISTS idx_swarms_active ON swarms(id, name) WHERE status = 'active'",
);
}
indexes.forEach((sql) => {
try {
db.exec(sql);
} catch (error) {
console.warn(`Warning: Could not create index: ${error.message}`);
}
});
}
/**
* Add performance tracking tables
*/
function addPerformanceTracking(db) {
// Agent performance metrics
db.exec(`
CREATE TABLE IF NOT EXISTS agent_performance (
agent_id TEXT PRIMARY KEY,
tasks_completed INTEGER DEFAULT 0,
tasks_failed INTEGER DEFAULT 0,
avg_completion_time REAL,
success_rate REAL,
last_updated DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (agent_id) REFERENCES agents(id)
)
`);
// Swarm performance metrics
db.exec(`
CREATE TABLE IF NOT EXISTS swarm_performance (
swarm_id TEXT PRIMARY KEY,
total_tasks INTEGER DEFAULT 0,
completed_tasks INTEGER DEFAULT 0,
avg_consensus_time REAL,
memory_efficiency REAL,
last_updated DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (swarm_id) REFERENCES swarms(id)
)
`);
// Create triggers to update performance metrics
db.exec(`
CREATE TRIGGER IF NOT EXISTS update_agent_performance
AFTER UPDATE OF status ON tasks
WHEN NEW.status = 'completed' OR NEW.status = 'failed'
BEGIN
INSERT OR REPLACE INTO agent_performance (agent_id, tasks_completed, tasks_failed)
VALUES (
NEW.agent_id,
COALESCE((SELECT tasks_completed FROM agent_performance WHERE agent_id = NEW.agent_id), 0) +
CASE WHEN NEW.status = 'completed' THEN 1 ELSE 0 END,
COALESCE((SELECT tasks_failed FROM agent_performance WHERE agent_id = NEW.agent_id), 0) +
CASE WHEN NEW.status = 'failed' THEN 1 ELSE 0 END
);
END
`);
}
/**
* Add memory optimization features
*/
function addMemoryOptimization(db) {
// Check if collective_memory table exists
const tables = db
.prepare(
`
SELECT name FROM sqlite_master
WHERE type='table' AND name = 'collective_memory'
`,
)
.all();
if (tables.length === 0) {
console.log('collective_memory table does not exist, skipping memory optimization');
return;
}
// Check and add access_count column
const hasAccessCount = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('collective_memory')
WHERE name = 'access_count'
`,
)
.get();
if (!hasAccessCount || hasAccessCount.count === 0) {
try {
db.exec(`
ALTER TABLE collective_memory
ADD COLUMN access_count INTEGER DEFAULT 0
`);
console.log('Added access_count column to collective_memory table');
} catch (error) {
if (!error.message.includes('duplicate column') && !error.message.includes('no such table')) {
throw error;
}
}
}
// Check and add accessed_at column (not last_accessed)
const hasAccessedAt = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('collective_memory')
WHERE name = 'accessed_at'
`,
)
.get();
if (!hasAccessedAt || hasAccessedAt.count === 0) {
try {
db.exec(`
ALTER TABLE collective_memory
ADD COLUMN accessed_at DATETIME
`);
console.log('Added accessed_at column to collective_memory table');
} catch (error) {
if (!error.message.includes('duplicate column') && !error.message.includes('no such table')) {
throw error;
}
}
}
// Add compressed and size columns if missing
const hasCompressed = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('collective_memory')
WHERE name = 'compressed'
`,
)
.get();
if (!hasCompressed || hasCompressed.count === 0) {
try {
db.exec(`
ALTER TABLE collective_memory
ADD COLUMN compressed INTEGER DEFAULT 0
`);
} catch (error) {
if (!error.message.includes('duplicate column') && !error.message.includes('no such table')) {
throw error;
}
}
}
const hasSize = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('collective_memory')
WHERE name = 'size'
`,
)
.get();
if (!hasSize || hasSize.count === 0) {
try {
db.exec(`
ALTER TABLE collective_memory
ADD COLUMN size INTEGER DEFAULT 0
`);
} catch (error) {
if (!error.message.includes('duplicate column') && !error.message.includes('no such table')) {
throw error;
}
}
}
// Create memory usage summary view
db.exec(`
CREATE VIEW IF NOT EXISTS memory_usage_summary AS
SELECT
swarm_id,
COUNT(*) as total_entries,
SUM(LENGTH(value)) as total_size,
AVG(access_count) as avg_access_count,
COUNT(CASE WHEN access_count = 0 THEN 1 END) as unused_entries
FROM collective_memory
GROUP BY swarm_id
`);
// Add memory cleanup tracking
db.exec(`
CREATE TABLE IF NOT EXISTS memory_cleanup_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
swarm_id TEXT,
entries_removed INTEGER,
space_reclaimed INTEGER,
cleanup_type TEXT,
performed_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`);
}
/**
* Add behavioral tracking features
*/
function addBehavioralTracking(db) {
// Agent interaction patterns
db.exec(`
CREATE TABLE IF NOT EXISTS agent_interactions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
from_agent_id TEXT,
to_agent_id TEXT,
interaction_type TEXT,
swarm_id TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (from_agent_id) REFERENCES agents(id),
FOREIGN KEY (to_agent_id) REFERENCES agents(id),
FOREIGN KEY (swarm_id) REFERENCES swarms(id)
)
`);
// Behavioral patterns
db.exec(`
CREATE TABLE IF NOT EXISTS behavioral_patterns (
id INTEGER PRIMARY KEY AUTOINCREMENT,
swarm_id TEXT,
pattern_type TEXT,
pattern_data TEXT, -- JSON
confidence REAL,
first_observed DATETIME DEFAULT CURRENT_TIMESTAMP,
last_observed DATETIME DEFAULT CURRENT_TIMESTAMP,
occurrence_count INTEGER DEFAULT 1,
FOREIGN KEY (swarm_id) REFERENCES swarms(id)
)
`);
// Create indexes for behavioral analysis
db.exec(`
CREATE INDEX IF NOT EXISTS idx_interactions_swarm ON agent_interactions(swarm_id);
CREATE INDEX IF NOT EXISTS idx_interactions_agents ON agent_interactions(from_agent_id, to_agent_id);
CREATE INDEX IF NOT EXISTS idx_patterns_swarm_type ON behavioral_patterns(swarm_id, pattern_type);
`);
}
/**
* Database maintenance utilities
*/
export async function performMaintenance(dbPath, options = {}) {
const spinner = ora('Performing database maintenance...').start();
try {
const db = new Database(dbPath);
// Clean up old memory entries
if (options.cleanMemory) {
// Check if collective_memory table exists
const hasMemoryTable = db
.prepare(
`
SELECT name FROM sqlite_master
WHERE type='table' AND name='collective_memory'
`,
)
.get();
if (hasMemoryTable) {
spinner.text = 'Cleaning old memory entries...';
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - (options.memoryRetentionDays || 30));
try {
const result = db
.prepare(
`
DELETE FROM collective_memory
WHERE accessed_at < ? AND access_count < 5
`,
)
.run(cutoffDate.toISOString());
console.log(chalk.green(`✓ Removed ${result.changes} old memory entries`));
} catch (error) {
console.warn(chalk.yellow(`⚠ Could not clean memory entries: ${error.message}`));
}
} else {
console.log(chalk.yellow('⚠ collective_memory table not found, skipping memory cleanup'));
}
}
// Archive completed tasks
if (options.archiveTasks) {
spinner.text = 'Archiving completed tasks...';
// Create archive table if not exists
db.exec(`
CREATE TABLE IF NOT EXISTS tasks_archive AS
SELECT * FROM tasks WHERE 1=0
`);
// Check if completed_at column exists
const hasCompletedAt = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('tasks')
WHERE name = 'completed_at'
`,
)
.get();
let archived = { changes: 0 };
if (hasCompletedAt && hasCompletedAt.count > 0) {
// Move old completed tasks using completed_at
const archiveCutoff = new Date();
archiveCutoff.setDate(archiveCutoff.getDate() - (options.taskRetentionDays || 7));
db.exec(`
INSERT INTO tasks_archive
SELECT * FROM tasks
WHERE status = 'completed' AND completed_at < '${archiveCutoff.toISOString()}'
`);
archived = db
.prepare(
`
DELETE FROM tasks
WHERE status = 'completed' AND completed_at < ?
`,
)
.run(archiveCutoff.toISOString());
} else {
// Use created_at as fallback
const archiveCutoff = new Date();
archiveCutoff.setDate(archiveCutoff.getDate() - (options.taskRetentionDays || 7));
db.exec(`
INSERT INTO tasks_archive
SELECT * FROM tasks
WHERE status = 'completed' AND created_at < '${archiveCutoff.toISOString()}'
`);
archived = db
.prepare(
`
DELETE FROM tasks
WHERE status = 'completed' AND created_at < ?
`,
)
.run(archiveCutoff.toISOString());
}
console.log(chalk.green(`✓ Archived ${archived.changes} completed tasks`));
}
// Update statistics
spinner.text = 'Updating database statistics...';
db.exec('ANALYZE');
// Check integrity
if (options.checkIntegrity) {
spinner.text = 'Checking database integrity...';
const integrityCheck = db.prepare('PRAGMA integrity_check').get();
if (integrityCheck.integrity_check === 'ok') {
console.log(chalk.green('✓ Database integrity check passed'));
} else {
console.log(chalk.yellow('⚠ Database integrity issues detected'));
}
}
db.close();
spinner.succeed('Database maintenance complete!');
} catch (error) {
spinner.fail('Database maintenance failed');
console.error(chalk.red('Error:'), error.message);
}
}
/**
* Generate optimization report
*/
export async function generateOptimizationReport(dbPath) {
try {
const db = new Database(dbPath, { readonly: true });
const report = {
schemaVersion: getSchemaVersion(db),
tables: {},
indexes: [],
performance: {},
};
// Get table statistics
const tables = db
.prepare(
`
SELECT name FROM sqlite_master WHERE type='table'
`,
)
.all();
for (const table of tables) {
const count = db.prepare(`SELECT COUNT(*) as count FROM ${table.name}`).get();
const size = db
.prepare(
`
SELECT SUM(pgsize) as size FROM dbstat WHERE name=?
`,
)
.get(table.name);
report.tables[table.name] = {
rowCount: count.count,
sizeBytes: size?.size || 0,
};
}
// Get index information
report.indexes = db
.prepare(
`
SELECT name, tbl_name FROM sqlite_master WHERE type='index'
`,
)
.all();
// Get performance metrics (check if completed_at column exists)
let avgTaskTime = { avg_minutes: 0 };
try {
// First check if completed_at column exists
const hasCompletedAt = db
.prepare(
`
SELECT COUNT(*) as count FROM pragma_table_info('tasks')
WHERE name = 'completed_at'
`,
)
.get();
if (hasCompletedAt && hasCompletedAt.count > 0) {
avgTaskTime = db
.prepare(
`
SELECT AVG(julianday(completed_at) - julianday(created_at)) * 24 * 60 as avg_minutes
FROM tasks WHERE completed_at IS NOT NULL
`,
)
.get();
}
} catch (error) {
// If error, just use default value
console.warn('Could not calculate average task time:', error.message);
}
report.performance.avgTaskCompletionMinutes = avgTaskTime?.avg_minutes || 0;
db.close();
return report;
} catch (error) {
console.error('Error generating report:', error);
return null;
}
}
// Export for use in CLI
export default {
optimizeHiveMindDatabase,
performMaintenance,
generateOptimizationReport,
};