claude-flow
Version:
Ruflo - Enterprise AI agent orchestration for Claude Code. Deploy 60+ specialized agents in coordinated swarms with self-learning, fault-tolerant consensus, vector memory, and MCP integration
1,353 lines (1,286 loc) • 95.2 kB
JavaScript
/**
* V3 Memory Initializer
* Properly initializes the memory database with sql.js (WASM SQLite)
* Includes pattern tables, vector embeddings, migration state tracking
*
* ADR-053: Routes through ControllerRegistry → AgentDB v3 when available,
* falls back to raw sql.js for backwards compatibility.
*
* @module v3/cli/memory-initializer
*/
import * as fs from 'fs';
import * as path from 'path';
import { readFileMaybeEncrypted, writeFileRestricted } from '../fs-secure.js';
/**
* #1854: previously every site that needed the memory directory hardcoded
* `getMemoryRoot()`, so the documented config entry
* points (`memory.persistPath` config field, `memory configure --path`,
* `CLAUDE_FLOW_MEMORY_PATH` env var) all silently no-op'd. This helper
* is the single source of truth — every `.swarm/memory.db` resolution in
* this file flows through it.
*
* Precedence (highest → lowest):
* 1. CLAUDE_FLOW_MEMORY_PATH env var
* 2. memory.persistPath / memory.path in claude-flow.config.json (cwd or
* the directory the CLI was invoked from)
* 3. Default: cwd/.swarm
*
* Cached per-process so repeated lookups are cheap; reset only by spawning
* a fresh process (which is how config changes already propagate).
*/
let _memoryRootCache;
export function getMemoryRoot() {
if (_memoryRootCache !== undefined)
return _memoryRootCache;
// 1. Env var
const envPath = process.env.CLAUDE_FLOW_MEMORY_PATH;
if (envPath && envPath.trim().length > 0) {
_memoryRootCache = path.resolve(envPath);
return _memoryRootCache;
}
// 2. Config file (claude-flow.config.json)
const configCandidates = [
path.resolve(process.cwd(), 'claude-flow.config.json'),
path.resolve(process.cwd(), '.claude-flow', 'config.json'),
];
for (const configPath of configCandidates) {
if (!fs.existsSync(configPath))
continue;
try {
const raw = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
const fromConfig = raw?.memory?.persistPath ?? raw?.memory?.path;
if (typeof fromConfig === 'string' && fromConfig.trim().length > 0) {
_memoryRootCache = path.resolve(fromConfig);
return _memoryRootCache;
}
}
catch {
/* malformed config — fall through to default */
}
}
// 3. Default
_memoryRootCache = path.resolve(process.cwd(), '.swarm');
return _memoryRootCache;
}
/** For tests + the `memory configure` flow that mutates the config at runtime. */
export function _resetMemoryRootCache() {
_memoryRootCache = undefined;
}
/**
* #2105: Resolve the full path to the SQLite memory database.
* Precedence (highest to lowest):
* 1. cliFlag - explicit --path flag passed by a subcommand
* 2. CLAUDE_FLOW_DB_PATH - full file-path override (new in #2105)
* 3. getMemoryRoot()/memory.db - directory from CLAUDE_FLOW_MEMORY_PATH /
* config / default cwd/.swarm
*/
export function resolveDbPath(cliFlag) {
if (cliFlag && cliFlag.trim().length > 0) {
return path.resolve(cliFlag);
}
const envDb = process.env.CLAUDE_FLOW_DB_PATH;
if (envDb && envDb.trim().length > 0) {
return path.resolve(envDb);
}
return path.join(getMemoryRoot(), 'memory.db');
}
// ADR-053: Lazy import of AgentDB v3 bridge
let _bridge;
async function getBridge() {
// #2120 — Allow callers to force the raw sql.js fallback path. The
// ensureSchemaColumns backfill (NULL → 'active') lives in that
// fallback, so smokes that verify legacy-DB migration semantics need a
// way to bypass the bridge. Also useful when the bridge would hang on
// network-bound init (Xenova model fetch) in offline CI.
if (process.env.CLAUDE_FLOW_DISABLE_BRIDGE === '1')
return null;
if (_bridge === null)
return null;
if (_bridge)
return _bridge;
try {
_bridge = await import('./memory-bridge.js');
return _bridge;
}
catch {
_bridge = null;
return null;
}
}
/**
* Enhanced schema with pattern confidence, temporal decay, versioning
* Vector embeddings enabled for semantic search
*/
export const MEMORY_SCHEMA_V3 = `
-- RuFlo V3 Memory Database
-- Version: 3.0.0
-- Features: Pattern learning, vector embeddings, temporal decay, migration tracking
PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
PRAGMA foreign_keys = ON;
-- ============================================
-- CORE MEMORY TABLES
-- ============================================
-- Memory entries (main storage)
CREATE TABLE IF NOT EXISTS memory_entries (
id TEXT PRIMARY KEY,
key TEXT NOT NULL,
namespace TEXT DEFAULT 'default',
content TEXT NOT NULL,
type TEXT DEFAULT 'semantic' CHECK(type IN ('semantic', 'episodic', 'procedural', 'working', 'pattern')),
-- Vector embedding for semantic search (stored as JSON array)
embedding TEXT,
embedding_model TEXT DEFAULT 'local',
embedding_dimensions INTEGER,
-- Metadata
tags TEXT, -- JSON array
metadata TEXT, -- JSON object
owner_id TEXT,
-- Timestamps
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
expires_at INTEGER,
last_accessed_at INTEGER,
-- Access tracking for hot/cold detection
access_count INTEGER DEFAULT 0,
-- Status
status TEXT DEFAULT 'active' CHECK(status IN ('active', 'archived', 'deleted')),
UNIQUE(namespace, key)
);
-- Indexes for memory entries
CREATE INDEX IF NOT EXISTS idx_memory_namespace ON memory_entries(namespace);
CREATE INDEX IF NOT EXISTS idx_memory_key ON memory_entries(key);
CREATE INDEX IF NOT EXISTS idx_memory_type ON memory_entries(type);
CREATE INDEX IF NOT EXISTS idx_memory_status ON memory_entries(status);
CREATE INDEX IF NOT EXISTS idx_memory_created ON memory_entries(created_at);
CREATE INDEX IF NOT EXISTS idx_memory_accessed ON memory_entries(last_accessed_at);
CREATE INDEX IF NOT EXISTS idx_memory_owner ON memory_entries(owner_id);
-- ============================================
-- PATTERN LEARNING TABLES
-- ============================================
-- Learned patterns with confidence scoring and versioning
CREATE TABLE IF NOT EXISTS patterns (
id TEXT PRIMARY KEY,
-- Pattern identification
name TEXT NOT NULL,
pattern_type TEXT NOT NULL CHECK(pattern_type IN (
'task-routing', 'error-recovery', 'optimization', 'learning',
'coordination', 'prediction', 'code-pattern', 'workflow'
)),
-- Pattern definition
condition TEXT NOT NULL, -- Regex or semantic match
action TEXT NOT NULL, -- What to do when pattern matches
description TEXT,
-- Confidence scoring (0.0 - 1.0)
confidence REAL DEFAULT 0.5,
success_count INTEGER DEFAULT 0,
failure_count INTEGER DEFAULT 0,
-- Temporal decay
decay_rate REAL DEFAULT 0.01, -- How fast confidence decays
half_life_days INTEGER DEFAULT 30, -- Days until confidence halves without use
-- Vector embedding for semantic pattern matching
embedding TEXT,
embedding_dimensions INTEGER,
-- Versioning
version INTEGER DEFAULT 1,
parent_id TEXT REFERENCES patterns(id),
-- Metadata
tags TEXT, -- JSON array
metadata TEXT, -- JSON object
source TEXT, -- Where the pattern was learned from
-- Timestamps
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
last_matched_at INTEGER,
last_success_at INTEGER,
last_failure_at INTEGER,
-- Status
status TEXT DEFAULT 'active' CHECK(status IN ('active', 'archived', 'deprecated', 'experimental'))
);
-- Indexes for patterns
CREATE INDEX IF NOT EXISTS idx_patterns_type ON patterns(pattern_type);
CREATE INDEX IF NOT EXISTS idx_patterns_confidence ON patterns(confidence DESC);
CREATE INDEX IF NOT EXISTS idx_patterns_status ON patterns(status);
CREATE INDEX IF NOT EXISTS idx_patterns_last_matched ON patterns(last_matched_at);
-- Pattern evolution history (for versioning)
CREATE TABLE IF NOT EXISTS pattern_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
pattern_id TEXT NOT NULL REFERENCES patterns(id),
version INTEGER NOT NULL,
-- Snapshot of pattern state
confidence REAL,
success_count INTEGER,
failure_count INTEGER,
condition TEXT,
action TEXT,
-- What changed
change_type TEXT CHECK(change_type IN ('created', 'updated', 'success', 'failure', 'decay', 'merged', 'split')),
change_reason TEXT,
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
);
CREATE INDEX IF NOT EXISTS idx_pattern_history_pattern ON pattern_history(pattern_id);
-- ============================================
-- LEARNING & TRAJECTORY TABLES
-- ============================================
-- Learning trajectories (SONA integration)
CREATE TABLE IF NOT EXISTS trajectories (
id TEXT PRIMARY KEY,
session_id TEXT,
-- Trajectory state
status TEXT DEFAULT 'active' CHECK(status IN ('active', 'completed', 'failed', 'abandoned')),
verdict TEXT CHECK(verdict IN ('success', 'failure', 'partial', NULL)),
-- Context
task TEXT,
context TEXT, -- JSON object
-- Metrics
total_steps INTEGER DEFAULT 0,
total_reward REAL DEFAULT 0,
-- Timestamps
started_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
ended_at INTEGER,
-- Reference to extracted pattern (if any)
extracted_pattern_id TEXT REFERENCES patterns(id)
);
-- Trajectory steps
CREATE TABLE IF NOT EXISTS trajectory_steps (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trajectory_id TEXT NOT NULL REFERENCES trajectories(id),
step_number INTEGER NOT NULL,
-- Step data
action TEXT NOT NULL,
observation TEXT,
reward REAL DEFAULT 0,
-- Metadata
metadata TEXT, -- JSON object
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
);
CREATE INDEX IF NOT EXISTS idx_steps_trajectory ON trajectory_steps(trajectory_id);
-- ============================================
-- MIGRATION STATE TRACKING
-- ============================================
-- Migration state (for resume capability)
CREATE TABLE IF NOT EXISTS migration_state (
id TEXT PRIMARY KEY,
migration_type TEXT NOT NULL, -- 'v2-to-v3', 'pattern', 'memory', etc.
-- Progress tracking
status TEXT DEFAULT 'pending' CHECK(status IN ('pending', 'in_progress', 'completed', 'failed', 'rolled_back')),
total_items INTEGER DEFAULT 0,
processed_items INTEGER DEFAULT 0,
failed_items INTEGER DEFAULT 0,
skipped_items INTEGER DEFAULT 0,
-- Current position (for resume)
current_batch INTEGER DEFAULT 0,
last_processed_id TEXT,
-- Source/destination info
source_path TEXT,
source_type TEXT,
destination_path TEXT,
-- Backup info
backup_path TEXT,
backup_created_at INTEGER,
-- Error tracking
last_error TEXT,
errors TEXT, -- JSON array of errors
-- Timestamps
started_at INTEGER,
completed_at INTEGER,
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
);
-- ============================================
-- SESSION MANAGEMENT
-- ============================================
-- Sessions for context persistence
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
-- Session state
state TEXT NOT NULL, -- JSON object with full session state
status TEXT DEFAULT 'active' CHECK(status IN ('active', 'paused', 'completed', 'expired')),
-- Context
project_path TEXT,
branch TEXT,
-- Metrics
tasks_completed INTEGER DEFAULT 0,
patterns_learned INTEGER DEFAULT 0,
-- Timestamps
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
expires_at INTEGER
);
-- ============================================
-- VECTOR INDEX METADATA (for HNSW)
-- ============================================
-- Track HNSW index state
CREATE TABLE IF NOT EXISTS vector_indexes (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
-- Index configuration
dimensions INTEGER NOT NULL,
metric TEXT DEFAULT 'cosine' CHECK(metric IN ('cosine', 'euclidean', 'dot')),
-- HNSW parameters
hnsw_m INTEGER DEFAULT 16,
hnsw_ef_construction INTEGER DEFAULT 200,
hnsw_ef_search INTEGER DEFAULT 100,
-- Quantization
quantization_type TEXT CHECK(quantization_type IN ('none', 'scalar', 'product')),
quantization_bits INTEGER DEFAULT 8,
-- Statistics
total_vectors INTEGER DEFAULT 0,
last_rebuild_at INTEGER,
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
);
-- ============================================
-- GRAPH EDGES (ADR-130 Phase 1)
-- Unified knowledge graph backend — sql.js canonical store
-- ============================================
-- Unified graph edges table (ADR-130)
-- Node IDs use domain-prefixed format: {domain}:{uuid}
-- where domain in (mem, agent, task, entity, span, pattern)
CREATE TABLE IF NOT EXISTS graph_edges (
id TEXT PRIMARY KEY, -- edge-{uuid}
source_id TEXT NOT NULL, -- domain-prefixed node ID
target_id TEXT NOT NULL, -- domain-prefixed node ID
relation TEXT NOT NULL, -- e.g. "caused", "depends-on", "imports"
weight REAL DEFAULT 1.0,
-- Temporal / reliability semantics (ADR-130 §"graph that forgets" property)
confidence REAL DEFAULT 1.0, -- [0,1]; updated by JUDGE step
decay_rate REAL DEFAULT 0.0, -- per-day exponential decay applied at read time
last_reinforced TEXT, -- ISO-8601; set when CONSOLIDATE re-touches edge
witness_id TEXT, -- FK to verification/witness-fixes.json (ADR-103)
-- Embedding storage: "inline:{base64}" | "vector_indexes:{id}" | NULL
embedding_ref TEXT,
metadata TEXT, -- JSON blob for plugin-specific fields
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_graph_edges_source ON graph_edges (source_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_target ON graph_edges (target_id);
CREATE INDEX IF NOT EXISTS idx_graph_edges_relation ON graph_edges (relation);
CREATE INDEX IF NOT EXISTS idx_graph_edges_reinforced ON graph_edges (last_reinforced);
-- ============================================
-- SYSTEM METADATA
-- ============================================
CREATE TABLE IF NOT EXISTS metadata (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at INTEGER DEFAULT (strftime('%s', 'now') * 1000)
);
`;
let hnswIndex = null;
let hnswInitializing = false;
/**
* Get or create the HNSW index singleton
* Lazily initializes from SQLite data on first use
*/
export async function getHNSWIndex(options) {
const dimensions = options?.dimensions ?? 384;
// Return existing index if already initialized
if (hnswIndex?.initialized && !options?.forceRebuild) {
return hnswIndex;
}
// Prevent concurrent initialization
if (hnswInitializing) {
// Wait for initialization to complete
while (hnswInitializing) {
await new Promise(resolve => setTimeout(resolve, 10));
}
return hnswIndex;
}
hnswInitializing = true;
try {
// Import @ruvector/core dynamically
// Handle both ESM (default export) and CJS patterns
const ruvectorModule = await import('@ruvector/core').catch(() => null);
if (!ruvectorModule) {
hnswInitializing = false;
return null; // HNSW not available
}
// ESM returns { default: { VectorDb, ... } }, CJS returns { VectorDb, ... }
const ruvectorCore = ruvectorModule.default || ruvectorModule;
if (!ruvectorCore?.VectorDb) {
hnswInitializing = false;
return null; // VectorDb not found
}
const { VectorDb } = ruvectorCore;
// Persistent storage paths — resolve to absolute to survive CWD changes
const swarmDir = getMemoryRoot();
if (!fs.existsSync(swarmDir)) {
fs.mkdirSync(swarmDir, { recursive: true });
}
const hnswPath = path.join(swarmDir, 'hnsw.index');
const metadataPath = path.join(swarmDir, 'hnsw.metadata.json');
const dbPath = options?.dbPath ? path.resolve(options.dbPath) : path.join(swarmDir, 'memory.db');
// Create HNSW index with persistent storage
// @ruvector/core uses string enum for distanceMetric: 'Cosine', 'Euclidean', 'DotProduct', 'Manhattan'
const db = new VectorDb({
dimensions,
distanceMetric: 'Cosine',
storagePath: hnswPath // Persistent storage!
});
// Load metadata (entry info) if exists
const entries = new Map();
if (fs.existsSync(metadataPath)) {
try {
const metadataJson = fs.readFileSync(metadataPath, 'utf-8');
const metadata = JSON.parse(metadataJson);
for (const [key, value] of metadata) {
entries.set(key, value);
}
}
catch {
// Metadata load failed, will rebuild
}
}
hnswIndex = {
db,
entries,
dimensions,
initialized: false
};
// Check if index already has data (from persistent storage)
const existingLen = await db.len();
if (existingLen > 0 && entries.size > 0) {
// Index loaded from disk, skip SQLite sync
hnswIndex.initialized = true;
hnswInitializing = false;
return hnswIndex;
}
if (fs.existsSync(dbPath)) {
try {
const initSqlJs = (await import('sql.js')).default;
const SQL = await initSqlJs();
const fileBuffer = readFileMaybeEncrypted(dbPath, null);
const sqlDb = new SQL.Database(fileBuffer);
// Load all entries with embeddings
const result = sqlDb.exec(`
SELECT id, key, namespace, content, embedding
FROM memory_entries
WHERE status = 'active' AND embedding IS NOT NULL
LIMIT 10000
`);
if (result[0]?.values) {
for (const row of result[0].values) {
const [id, key, ns, content, embeddingJson] = row;
if (embeddingJson) {
try {
const embedding = JSON.parse(embeddingJson);
const vector = new Float32Array(embedding);
await db.insert({
id: String(id),
vector
});
hnswIndex.entries.set(String(id), {
id: String(id),
key: key || String(id),
namespace: ns || 'default',
content: content || ''
});
}
catch {
// Skip invalid embeddings
}
}
}
}
sqlDb.close();
}
catch {
// SQLite load failed, start with empty index
}
}
hnswIndex.initialized = true;
hnswInitializing = false;
return hnswIndex;
}
catch {
hnswInitializing = false;
return null;
}
}
/**
* Save HNSW metadata to disk for persistence
*/
function saveHNSWMetadata() {
if (!hnswIndex?.entries)
return;
try {
const swarmDir = getMemoryRoot();
const metadataPath = path.join(swarmDir, 'hnsw.metadata.json');
const metadata = Array.from(hnswIndex.entries.entries());
fs.writeFileSync(metadataPath, JSON.stringify(metadata));
}
catch {
// Silently fail - metadata save is best-effort
}
}
/**
* Add entry to HNSW index (with automatic persistence)
*/
export async function addToHNSWIndex(id, embedding, entry) {
// ADR-053: Try AgentDB v3 bridge first
const bridge = await getBridge();
if (bridge) {
const bridgeResult = await bridge.bridgeAddToHNSW(id, embedding, entry);
if (bridgeResult === true)
return true;
}
const index = await getHNSWIndex({ dimensions: embedding.length });
if (!index)
return false;
try {
const vector = new Float32Array(embedding);
await index.db.insert({
id,
vector
});
index.entries.set(id, entry);
// Save metadata for persistence (debounced would be better for high-volume)
saveHNSWMetadata();
return true;
}
catch {
return false;
}
}
/**
* Search HNSW index (150x faster than brute-force)
* Returns results sorted by similarity (highest first)
*/
export async function searchHNSWIndex(queryEmbedding, options) {
// ADR-053: Try AgentDB v3 bridge first
const bridge = await getBridge();
if (bridge) {
const bridgeResult = await bridge.bridgeSearchHNSW(queryEmbedding, options);
if (bridgeResult)
return bridgeResult;
}
const index = await getHNSWIndex({ dimensions: queryEmbedding.length });
if (!index)
return null;
try {
const vector = new Float32Array(queryEmbedding);
const k = options?.k ?? 10;
// HNSW search returns results with cosine distance (lower = more similar)
const results = await index.db.search({ vector, k: k * 2 }); // Get extra for filtering
const filtered = [];
for (const result of results) {
const entry = index.entries.get(result.id);
if (!entry)
continue;
// Filter by namespace if specified
if (options?.namespace && options.namespace !== 'all' && entry.namespace !== options.namespace) {
continue;
}
// Convert cosine distance to similarity score (1 - distance)
// Cosine distance from @ruvector/core: 0 = identical, 2 = opposite
const score = 1 - (result.score / 2);
filtered.push({
id: entry.id.substring(0, 12),
key: entry.key || entry.id.substring(0, 15),
content: entry.content.substring(0, 60) + (entry.content.length > 60 ? '...' : ''),
score,
namespace: entry.namespace
});
if (filtered.length >= k)
break;
}
// Sort by score descending (highest similarity first)
filtered.sort((a, b) => b.score - a.score);
return filtered;
}
catch {
return null;
}
}
/**
* Get HNSW index status
*/
export function getHNSWStatus() {
// ADR-053: If bridge was previously loaded, report availability
if (_bridge && _bridge !== null) {
// Bridge is loaded — HNSW-equivalent is available via AgentDB v3
return {
available: true,
initialized: true,
entryCount: hnswIndex?.entries.size ?? 0,
dimensions: hnswIndex?.dimensions ?? 384
};
}
return {
available: hnswIndex !== null,
initialized: hnswIndex?.initialized ?? false,
entryCount: hnswIndex?.entries.size ?? 0,
dimensions: hnswIndex?.dimensions ?? 384
};
}
/**
* Clear the HNSW index (for rebuilding)
*/
export function clearHNSWIndex() {
hnswIndex = null;
}
/**
* Invalidate the in-memory HNSW cache so the next search rebuilds from DB.
* Call this after deleting entries that had embeddings to prevent ghost
* vectors from appearing in search results.
*/
export function rebuildSearchIndex() {
hnswIndex = null;
hnswInitializing = false;
}
// ============================================================================
// INT8 VECTOR QUANTIZATION (4x memory reduction)
// ============================================================================
/**
* Quantize a Float32 embedding to Int8 (4x memory reduction)
* Uses symmetric quantization with scale factor stored per-vector
*
* @param embedding - Float32 embedding array
* @returns Quantized Int8 array with scale factor
*/
export function quantizeInt8(embedding) {
const arr = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
// Find min/max for symmetric quantization
let min = Infinity, max = -Infinity;
for (let i = 0; i < arr.length; i++) {
if (arr[i] < min)
min = arr[i];
if (arr[i] > max)
max = arr[i];
}
// Symmetric quantization: scale = max(|min|, |max|) / 127
const absMax = Math.max(Math.abs(min), Math.abs(max));
const scale = absMax / 127 || 1e-10; // Avoid division by zero
const zeroPoint = 0; // Symmetric quantization
// Quantize
const quantized = new Int8Array(arr.length);
for (let i = 0; i < arr.length; i++) {
// Clamp to [-127, 127] to leave room for potential rounding
const q = Math.round(arr[i] / scale);
quantized[i] = Math.max(-127, Math.min(127, q));
}
return { quantized, scale, zeroPoint };
}
/**
* Dequantize Int8 back to Float32
*
* @param quantized - Int8 quantized array
* @param scale - Scale factor from quantization
* @param zeroPoint - Zero point (usually 0 for symmetric)
* @returns Float32Array
*/
export function dequantizeInt8(quantized, scale, zeroPoint = 0) {
const result = new Float32Array(quantized.length);
for (let i = 0; i < quantized.length; i++) {
result[i] = (quantized[i] - zeroPoint) * scale;
}
return result;
}
/**
* Compute cosine similarity between quantized vectors
* Faster than dequantizing first
*/
export function quantizedCosineSim(a, aScale, b, bScale) {
if (a.length !== b.length)
return 0;
let dot = 0, normA = 0, normB = 0;
for (let i = 0; i < a.length; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
// Scales cancel out in cosine similarity for normalized vectors
const mag = Math.sqrt(normA * normB);
return mag === 0 ? 0 : dot / mag;
}
/**
* Get quantization statistics for an embedding
*/
export function getQuantizationStats(embedding) {
const len = embedding.length;
const originalBytes = len * 4; // Float32 = 4 bytes
const quantizedBytes = len + 8; // Int8 = 1 byte + 8 bytes for scale/zeroPoint
const compressionRatio = originalBytes / quantizedBytes;
return { originalBytes, quantizedBytes, compressionRatio };
}
// ============================================================================
// FLASH ATTENTION-STYLE BATCH OPERATIONS (V8-Optimized)
// ============================================================================
/**
* Batch cosine similarity - compute query against multiple vectors
* Optimized for V8 JIT with typed arrays
* ~50μs per 1000 vectors (384-dim)
*/
export function batchCosineSim(query, vectors) {
const n = vectors.length;
const scores = new Float32Array(n);
if (n === 0 || query.length === 0)
return scores;
// Pre-compute query norm
let queryNorm = 0;
for (let i = 0; i < query.length; i++) {
queryNorm += query[i] * query[i];
}
queryNorm = Math.sqrt(queryNorm);
if (queryNorm === 0)
return scores;
// Compute similarities
for (let v = 0; v < n; v++) {
const vec = vectors[v];
const len = Math.min(query.length, vec.length);
let dot = 0, vecNorm = 0;
for (let i = 0; i < len; i++) {
dot += query[i] * vec[i];
vecNorm += vec[i] * vec[i];
}
vecNorm = Math.sqrt(vecNorm);
scores[v] = vecNorm === 0 ? 0 : dot / (queryNorm * vecNorm);
}
return scores;
}
/**
* Softmax normalization for attention scores
* Numerically stable implementation
*/
export function softmaxAttention(scores, temperature = 1.0) {
const n = scores.length;
const result = new Float32Array(n);
if (n === 0)
return result;
// Find max for numerical stability
let max = scores[0];
for (let i = 1; i < n; i++) {
if (scores[i] > max)
max = scores[i];
}
// Compute exp and sum
let sum = 0;
for (let i = 0; i < n; i++) {
result[i] = Math.exp((scores[i] - max) / temperature);
sum += result[i];
}
// Normalize
if (sum > 0) {
for (let i = 0; i < n; i++) {
result[i] /= sum;
}
}
return result;
}
/**
* Top-K selection with partial sort (O(n + k log k))
* More efficient than full sort for small k
*/
export function topKIndices(scores, k) {
const n = scores.length;
if (k >= n) {
// Return all indices sorted by score
return Array.from({ length: n }, (_, i) => i)
.sort((a, b) => scores[b] - scores[a]);
}
// Build min-heap of size k
const heap = [];
for (let i = 0; i < n; i++) {
if (heap.length < k) {
heap.push({ idx: i, score: scores[i] });
// Bubble up
let j = heap.length - 1;
while (j > 0) {
const parent = Math.floor((j - 1) / 2);
if (heap[j].score < heap[parent].score) {
[heap[j], heap[parent]] = [heap[parent], heap[j]];
j = parent;
}
else
break;
}
}
else if (scores[i] > heap[0].score) {
// Replace min and heapify down
heap[0] = { idx: i, score: scores[i] };
let j = 0;
while (true) {
const left = 2 * j + 1, right = 2 * j + 2;
let smallest = j;
if (left < k && heap[left].score < heap[smallest].score)
smallest = left;
if (right < k && heap[right].score < heap[smallest].score)
smallest = right;
if (smallest === j)
break;
[heap[j], heap[smallest]] = [heap[smallest], heap[j]];
j = smallest;
}
}
}
// Extract and sort descending
return heap.sort((a, b) => b.score - a.score).map(h => h.idx);
}
/**
* Flash Attention-style search
* Combines batch similarity, softmax, and top-k in one pass
* Returns indices and attention weights
*/
export function flashAttentionSearch(query, vectors, options = {}) {
const { k = 10, temperature = 1.0, threshold = 0 } = options;
// Compute batch similarity
const scores = batchCosineSim(query, vectors);
// Get top-k indices
const indices = topKIndices(scores, k);
// Filter by threshold
const filtered = indices.filter(i => scores[i] >= threshold);
// Extract scores for filtered results
const topScores = new Float32Array(filtered.length);
for (let i = 0; i < filtered.length; i++) {
topScores[i] = scores[filtered[i]];
}
// Compute attention weights (softmax over top-k)
const weights = softmaxAttention(topScores, temperature);
return { indices: filtered, scores: topScores, weights };
}
// ============================================================================
// METADATA AND INITIALIZATION
// ============================================================================
/**
* Initial metadata to insert after schema creation
*/
export function getInitialMetadata(backend) {
return `
INSERT OR REPLACE INTO metadata (key, value) VALUES
('schema_version', '3.0.0'),
('backend', '${backend}'),
('created_at', '${new Date().toISOString()}'),
('sql_js', 'true'),
('vector_embeddings', 'enabled'),
('pattern_learning', 'enabled'),
('temporal_decay', 'enabled'),
('hnsw_indexing', 'enabled');
-- Create default vector index configuration. Dimension matches the default
-- ONNX embedding model (Xenova/all-MiniLM-L6-v2, 384-dim); HNSW rejects
-- inserts whose dim does not match this row, so a 768 here breaks every
-- memory_store --vector and memory_search on a fresh install (#1947).
INSERT OR IGNORE INTO vector_indexes (id, name, dimensions) VALUES
('default', 'default', 384),
('patterns', 'patterns', 384);
`;
}
/**
* Ensure memory_entries table has all required columns
* Adds missing columns for older databases (e.g., 'content' column)
*/
export async function ensureSchemaColumns(dbPath) {
const columnsAdded = [];
try {
if (!fs.existsSync(dbPath)) {
return { success: true, columnsAdded: [] };
}
const initSqlJs = (await import('sql.js')).default;
const SQL = await initSqlJs();
const fileBuffer = readFileMaybeEncrypted(dbPath, null);
const db = new SQL.Database(fileBuffer);
// Get current columns in memory_entries
const tableInfo = db.exec("PRAGMA table_info(memory_entries)");
const existingColumns = new Set(tableInfo[0]?.values?.map(row => row[1]) || []);
// Required columns that may be missing in older schemas
// Issue #977: 'type' column was missing from this list, causing store failures on older DBs
const requiredColumns = [
{ name: 'content', definition: "content TEXT DEFAULT ''" },
{ name: 'type', definition: "type TEXT DEFAULT 'semantic'" },
{ name: 'embedding', definition: 'embedding TEXT' },
{ name: 'embedding_model', definition: "embedding_model TEXT DEFAULT 'local'" },
{ name: 'embedding_dimensions', definition: 'embedding_dimensions INTEGER' },
{ name: 'tags', definition: 'tags TEXT' },
{ name: 'metadata', definition: 'metadata TEXT' },
{ name: 'owner_id', definition: 'owner_id TEXT' },
{ name: 'expires_at', definition: 'expires_at INTEGER' },
{ name: 'last_accessed_at', definition: 'last_accessed_at INTEGER' },
{ name: 'access_count', definition: 'access_count INTEGER DEFAULT 0' },
{ name: 'status', definition: "status TEXT DEFAULT 'active'" }
];
let modified = false;
for (const col of requiredColumns) {
if (!existingColumns.has(col.name)) {
try {
db.run(`ALTER TABLE memory_entries ADD COLUMN ${col.definition}`);
columnsAdded.push(col.name);
modified = true;
}
catch (e) {
// Column might already exist or other error - continue
}
}
}
// #2120 — Belt-and-suspenders backfill. `ALTER TABLE ADD COLUMN
// status TEXT DEFAULT 'active'` should populate existing rows with
// 'active' in modern SQLite, but: (a) some auto-memory bridge writes
// happen via INSERT paths that pass an explicit NULL, (b) some
// historical sql.js builds skipped the DEFAULT backfill, (c)
// entries can be migrated in from older snapshots. After ensuring
// the column exists, force-backfill any remaining NULL → 'active'.
// Safe on already-correct DBs (0 rows updated).
if (columnsAdded.includes('status') || existingColumns.has('status')) {
try {
db.run(`UPDATE memory_entries SET status = 'active' WHERE status IS NULL`);
modified = true;
}
catch {
/* table is read-only or doesn't exist — skip */
}
}
if (modified) {
// Save updated database
const data = db.export();
writeFileRestricted(dbPath, Buffer.from(data), { encrypt: true });
}
db.close();
return { success: true, columnsAdded };
}
catch (error) {
return {
success: false,
columnsAdded,
error: error instanceof Error ? error.message : String(error)
};
}
}
/**
* Check for legacy database installations and migrate if needed
*/
export async function checkAndMigrateLegacy(options) {
const { dbPath, verbose = false } = options;
// Check for legacy locations
const legacyPaths = [
path.join(process.cwd(), 'memory.db'),
path.join(process.cwd(), '.claude/memory.db'),
path.join(process.cwd(), 'data/memory.db'),
path.join(process.cwd(), '.claude-flow/memory.db')
];
for (const legacyPath of legacyPaths) {
if (fs.existsSync(legacyPath) && legacyPath !== dbPath) {
try {
const initSqlJs = (await import('sql.js')).default;
const SQL = await initSqlJs();
const legacyBuffer = fs.readFileSync(legacyPath);
const legacyDb = new SQL.Database(legacyBuffer);
// Check if it has data
const countResult = legacyDb.exec('SELECT COUNT(*) FROM memory_entries');
const count = countResult[0]?.values[0]?.[0] || 0;
// Get version if available
let version = 'unknown';
try {
const versionResult = legacyDb.exec("SELECT value FROM metadata WHERE key='schema_version'");
version = versionResult[0]?.values[0]?.[0] || 'unknown';
}
catch { /* no metadata table */ }
legacyDb.close();
if (count > 0) {
return {
needsMigration: true,
legacyVersion: version,
legacyEntries: count
};
}
}
catch {
// Not a valid SQLite database, skip
}
}
}
return { needsMigration: false };
}
/**
* ADR-053: Activate ControllerRegistry so AgentDB v3 controllers
* (ReasoningBank, SkillLibrary, ExplainableRecall, etc.) are instantiated.
*
* Uses the memory-bridge's getControllerRegistry() which lazily creates
* a singleton ControllerRegistry and initializes it with the given dbPath.
* After this call, all enabled controllers are ready for immediate use.
*
* Failures are isolated: if @claude-flow/memory or agentdb is not installed,
* this returns an empty result without throwing.
*/
async function activateControllerRegistry(dbPath, verbose) {
const startTime = performance.now();
const activated = [];
const failed = [];
try {
const bridge = await getBridge();
if (!bridge) {
return { activated, failed, initTimeMs: performance.now() - startTime };
}
const registry = await bridge.getControllerRegistry(dbPath);
if (!registry) {
return { activated, failed, initTimeMs: performance.now() - startTime };
}
// Collect controller status from the registry
if (typeof registry.listControllers === 'function') {
const controllers = registry.listControllers();
for (const ctrl of controllers) {
if (ctrl.enabled) {
activated.push(ctrl.name);
}
else {
failed.push(ctrl.name);
}
}
}
if (verbose && activated.length > 0) {
console.log(`ControllerRegistry: ${activated.length} controllers activated`);
}
}
catch {
// ControllerRegistry activation is best-effort
}
return { activated, failed, initTimeMs: performance.now() - startTime };
}
/**
* Initialize the memory database properly using sql.js
*/
export async function initializeMemoryDatabase(options) {
const { backend = 'hybrid', dbPath: customPath, force = false, verbose = false, migrate = true } = options;
const swarmDir = getMemoryRoot();
const dbPath = customPath || path.join(swarmDir, 'memory.db');
const dbDir = path.dirname(dbPath);
try {
// Create directory if needed
if (!fs.existsSync(dbDir)) {
fs.mkdirSync(dbDir, { recursive: true });
}
// Check for legacy installations
if (migrate) {
const legacyCheck = await checkAndMigrateLegacy({ dbPath, verbose });
if (legacyCheck.needsMigration && verbose) {
console.log(`Found legacy database (v${legacyCheck.legacyVersion}) with ${legacyCheck.legacyEntries} entries`);
}
}
// Check existing database
// #1791.6 — Idempotent re-init: if the database already exists and the
// caller did not pass --force, treat it as a successful no-op instead of
// an error. Callers (CLI, MCP tools, embeddings) can branch on
// `alreadyExists` if they want a different message; previous behavior
// surfaced an `[ERROR]` and a "Initialization failed" spinner even when
// the existing DB was perfectly healthy.
if (fs.existsSync(dbPath) && !force) {
return {
success: true,
alreadyExists: true,
backend,
dbPath,
schemaVersion: '3.0.0',
tablesCreated: [],
indexesCreated: [],
features: {
vectorEmbeddings: false,
patternLearning: false,
temporalDecay: false,
hnswIndexing: false,
migrationTracking: false
}
};
}
// Try to use sql.js (WASM SQLite)
let db;
let usedSqlJs = false;
try {
// Dynamic import of sql.js
const initSqlJs = (await import('sql.js')).default;
const SQL = await initSqlJs();
// Load existing database or create new
if (fs.existsSync(dbPath) && force) {
fs.unlinkSync(dbPath);
}
db = new SQL.Database();
usedSqlJs = true;
}
catch (e) {
// sql.js not available, fall back to writing schema file
if (verbose) {
console.log('sql.js not available, writing schema file for later initialization');
}
}
if (usedSqlJs && db) {
// Execute schema
db.run(MEMORY_SCHEMA_V3);
// Insert initial metadata
db.run(getInitialMetadata(backend));
// Save to file
const data = db.export();
const buffer = Buffer.from(data);
writeFileRestricted(dbPath, buffer, { encrypt: true });
// Close database
db.close();
// Also create schema file for reference
const schemaPath = path.join(dbDir, 'schema.sql');
fs.writeFileSync(schemaPath, MEMORY_SCHEMA_V3 + '\n' + getInitialMetadata(backend));
// ADR-053: Activate ControllerRegistry so controllers (ReasoningBank,
// SkillLibrary, ExplainableRecall, etc.) are instantiated during init
const controllerResult = await activateControllerRegistry(dbPath, verbose);
return {
success: true,
backend,
dbPath,
schemaVersion: '3.0.0',
tablesCreated: [
'memory_entries',
'patterns',
'pattern_history',
'trajectories',
'trajectory_steps',
'migration_state',
'sessions',
'vector_indexes',
'metadata'
],
indexesCreated: [
'idx_memory_namespace',
'idx_memory_key',
'idx_memory_type',
'idx_memory_status',
'idx_memory_created',
'idx_memory_accessed',
'idx_memory_owner',
'idx_patterns_type',
'idx_patterns_confidence',
'idx_patterns_status',
'idx_patterns_last_matched',
'idx_pattern_history_pattern',
'idx_steps_trajectory'
],
features: {
vectorEmbeddings: true,
patternLearning: true,
temporalDecay: true,
hnswIndexing: true,
migrationTracking: true
},
controllers: controllerResult,
};
}
else {
// Fall back to schema file approach
const schemaPath = path.join(dbDir, 'schema.sql');
fs.writeFileSync(schemaPath, MEMORY_SCHEMA_V3 + '\n' + getInitialMetadata(backend));
// Create minimal valid SQLite file
const sqliteHeader = Buffer.alloc(4096, 0);
// SQLite format 3 header
Buffer.from('SQLite format 3\0').copy(sqliteHeader, 0);
sqliteHeader[16] = 0x10; // page size high byte (4096)
sqliteHeader[17] = 0x00; // page size low byte
sqliteHeader[18] = 0x01; // file format write version
sqliteHeader[19] = 0x01; // file format read version
sqliteHeader[24] = 0x00; // max embedded payload
sqliteHeader[25] = 0x40;
sqliteHeader[26] = 0x20; // min embedded payload
sqliteHeader[27] = 0x20; // leaf payload
writeFileRestricted(dbPath, sqliteHeader, { encrypt: true });
// ADR-053: Activate ControllerRegistry even on fallback path
const controllerResult = await activateControllerRegistry(dbPath, verbose);
return {
success: true,
backend,
dbPath,
schemaVersion: '3.0.0',
tablesCreated: [
'memory_entries (pending)',
'patterns (pending)',
'pattern_history (pending)',
'trajectories (pending)',
'trajectory_steps (pending)',
'migration_state (pending)',
'sessions (pending)',
'vector_indexes (pending)',
'metadata (pending)'
],
indexesCreated: [],
features: {
vectorEmbeddings: true,
patternLearning: true,
temporalDecay: true,
hnswIndexing: true,
migrationTracking: true
},
controllers: controllerResult,
};
}
}
catch (error) {
return {
success: false,
backend,
dbPath,
schemaVersion: '3.0.0',
tablesCreated: [],
indexesCreated: [],
features: {
vectorEmbeddings: false,
patternLearning: false,
temporalDecay: false,
hnswIndexing: false,
migrationTracking: false
},
error: error instanceof Error ? error.message : String(error)
};
}
}
/**
* Check if memory database is properly initialized
*/
export async function checkMemoryInitialization(dbPath) {
const swarmDir = getMemoryRoot();
const path_ = dbPath || path.join(swarmDir, 'memory.db');
if (!fs.existsSync(path_)) {
return { initialized: false };
}
try {
// Try to load with sql.js
const initSqlJs = (await import('sql.js')).default;
const SQL = await initSqlJs();
const fileBuffer = fs.readFileSync(path_);
const db = new SQL.Database(fileBuffer);
// Check for metadata table
const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'");
const tableNames = tables[0]?.values?.map(v => v[0]) || [];
// Get version
let version = 'unknown';
let backend = 'unknown';
try {
const versionResult = db.exec("SELECT value FROM metadata WHERE key='schema_version'");
version = versionResult[0]?.values[0]?.[0] || 'unknown';
const backendResult = db.exec("SELECT value FROM metadata WHERE key='backend'");
backend = backendResult[0]?.values[0]?.[0] || 'unknown';
}
catch {
// Metadata table might not exist
}
db.close();
return {
initialized: true,
version,
backend,
features: {
vectorEmbeddings: tableNames.includes('vector_indexes'),
patternLearning: tableNames.includes('patterns'),
temporalDecay: tableNames.includes('pattern_history')
},
tables: tableNames
};
}
catch {
// Could not read database
return { initialized: false };
}
}
/**
* Apply temporal decay to patterns
* Reduces confidence of patterns that haven't been used recently
*/
export async function applyTemporalDecay(dbPath) {
const swarmDir = getMemoryRoot();
const path_ = dbPath || path.join(swarmDir, 'memory.db');
try {
const initSqlJs = (await import('sql.js')).default;
const SQL = await initSqlJs();
const fileBuffer = fs.readFileSync(path_);
const db = new SQL.Database(fileBuffer);
// Apply decay: confidence *= exp(-decay_rate * days_since_last_use)
const now = Date.now();