@stackmemoryai/stackmemory
Version:
Lossless, project-scoped memory for AI coding tools. Durable context across sessions with 56 MCP tools, FTS5 search, conductor orchestrator, loop/watch monitoring, snapshot capture, pre-flight overlap checks, Claude/Codex/OpenCode wrappers, Linear sync, a
1,472 lines • 58.7 kB
JavaScript
import { fileURLToPath as __fileURLToPath } from 'url';
import { dirname as __pathDirname } from 'path';
const __filename = __fileURLToPath(import.meta.url);
const __dirname = __pathDirname(__filename);
import Database from "better-sqlite3";
import {
FeatureAwareDatabaseAdapter
} from "./database-adapter.js";
import { logger } from "../monitoring/logger.js";
import { DatabaseError, ErrorCode, ValidationError } from "../errors/index.js";
import { FrameDatabase } from "../context/frame-database.js";
import * as fs from "fs/promises";
import * as path from "path";
class SQLiteAdapter extends FeatureAwareDatabaseAdapter {
db = null;
dbPath;
inTransactionFlag = false;
ftsEnabled = false;
vecEnabled = false;
embeddingProvider;
embeddingDimension;
constructor(projectId, config) {
super(projectId, config);
this.dbPath = config.dbPath;
this.embeddingProvider = config.embeddingProvider;
this.embeddingDimension = config.embeddingDimension || 384;
}
getEmbeddingProvider() {
return this.embeddingProvider;
}
getFeatures() {
return {
supportsFullTextSearch: this.ftsEnabled,
supportsVectorSearch: this.vecEnabled,
supportsPartitioning: false,
supportsAnalytics: false,
supportsCompression: false,
supportsMaterializedViews: false,
supportsParallelQueries: false
};
}
async connect() {
if (this.db) return;
const config = this.config;
const dir = path.dirname(this.dbPath);
await fs.mkdir(dir, { recursive: true });
this.db = new Database(this.dbPath);
this.db.pragma("foreign_keys = ON");
if (config.walMode !== false) {
this.db.pragma("journal_mode = WAL");
}
this.db.pragma("mmap_size = 268435456");
if (config.busyTimeout) {
this.db.pragma(`busy_timeout = ${config.busyTimeout}`);
}
if (config.cacheSize) {
this.db.pragma(`cache_size = ${config.cacheSize}`);
} else {
this.db.pragma("cache_size = -64000");
}
if (config.synchronous) {
this.db.pragma(`synchronous = ${config.synchronous}`);
}
logger.info("SQLite database connected", { dbPath: this.dbPath });
}
async disconnect() {
if (!this.db) return;
this.db.close();
this.db = null;
logger.info("SQLite database disconnected");
}
/**
* Get raw database handle for testing purposes
* @internal
*/
getRawDatabase() {
return this.db;
}
isConnected() {
return this.db !== null && this.db.open;
}
async ping() {
if (!this.db) return false;
try {
this.db.prepare("SELECT 1").get();
return true;
} catch (error) {
logger.debug("Database ping failed", {
error: error instanceof Error ? error.message : String(error)
});
return false;
}
}
async initializeSchema() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const frameDb = new FrameDatabase(this.db);
frameDb.initSchema();
try {
this.ensureCascadeConstraints();
} catch (e) {
logger.warn("Failed to ensure cascade constraints", e);
}
this.initializeFts();
this.initializeVec();
}
/**
* Ensure ON DELETE CASCADE exists for events/anchors referencing frames
* Migrates existing tables in-place if needed without data loss.
*/
ensureCascadeConstraints() {
if (!this.db) return;
const needsCascade = (table) => {
const rows = this.db.prepare(
`PRAGMA foreign_key_list(${table})`
).all();
return rows.some(
(r) => r.table === "frames" && String(r.on_delete).toUpperCase() !== "CASCADE"
);
};
const hasColumn = (table, column) => {
const cols = this.db.prepare(
`PRAGMA table_info(${table})`
).all();
return cols.some((c) => c.name === column);
};
const migrateTable = (table) => {
const createSql = table === "events" ? `CREATE TABLE events_new (
event_id TEXT PRIMARY KEY,
run_id TEXT NOT NULL,
frame_id TEXT NOT NULL,
seq INTEGER NOT NULL,
event_type TEXT NOT NULL,
payload TEXT NOT NULL,
ts INTEGER DEFAULT (unixepoch()),
FOREIGN KEY(frame_id) REFERENCES frames(frame_id) ON DELETE CASCADE
);` : `CREATE TABLE anchors_new (
anchor_id TEXT PRIMARY KEY,
frame_id TEXT NOT NULL,
project_id TEXT NOT NULL DEFAULT '',
type TEXT NOT NULL,
text TEXT NOT NULL,
priority INTEGER DEFAULT 0,
created_at INTEGER DEFAULT (unixepoch()),
metadata TEXT DEFAULT '{}',
FOREIGN KEY(frame_id) REFERENCES frames(frame_id) ON DELETE CASCADE
);`;
let selectCols;
if (table === "anchors") {
const hasProjectId = hasColumn("anchors", "project_id");
selectCols = hasProjectId ? "anchor_id, frame_id, project_id, type, text, priority, created_at, metadata" : `anchor_id, frame_id, '${this.projectId}' as project_id, type, text, priority, created_at, metadata`;
} else {
selectCols = "event_id, run_id, frame_id, seq, event_type, payload, ts";
}
const cols = table === "events" ? "event_id, run_id, frame_id, seq, event_type, payload, ts" : "anchor_id, frame_id, project_id, type, text, priority, created_at, metadata";
const idxSql = table === "events" ? [
"CREATE INDEX IF NOT EXISTS idx_events_frame ON events(frame_id);",
"CREATE INDEX IF NOT EXISTS idx_events_seq ON events(frame_id, seq);"
] : [
"CREATE INDEX IF NOT EXISTS idx_anchors_frame ON anchors(frame_id);"
];
this.db.exec("PRAGMA foreign_keys = OFF;");
this.db.exec("BEGIN;");
this.db.exec(createSql);
this.db.prepare(
`INSERT INTO ${table === "events" ? "events_new" : "anchors_new"} (${cols}) SELECT ${selectCols} FROM ${table}`
).run();
this.db.exec(`DROP TABLE ${table};`);
this.db.exec(`ALTER TABLE ${table}_new RENAME TO ${table};`);
for (const stmt of idxSql) this.db.exec(stmt);
this.db.exec("COMMIT;");
this.db.exec("PRAGMA foreign_keys = ON;");
logger.info(`Migrated ${table} to include ON DELETE CASCADE`);
};
if (needsCascade("events")) migrateTable("events");
if (needsCascade("anchors")) migrateTable("anchors");
}
/**
* Initialize FTS5 virtual table and sync triggers
*/
initializeFts() {
if (!this.db) return;
try {
this.db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS frames_fts USING fts5(
name, digest_text, inputs, outputs,
content='frames', content_rowid='rowid'
);
`);
this.db.exec(`
CREATE TRIGGER IF NOT EXISTS frames_ai AFTER INSERT ON frames BEGIN
INSERT INTO frames_fts(rowid, name, digest_text, inputs, outputs)
VALUES (new.rowid, new.name, new.digest_text, new.inputs, new.outputs);
END;
`);
this.db.exec(`
CREATE TRIGGER IF NOT EXISTS frames_ad AFTER DELETE ON frames BEGIN
INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)
VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);
END;
`);
this.db.exec(`
CREATE TRIGGER IF NOT EXISTS frames_au AFTER UPDATE ON frames BEGIN
INSERT INTO frames_fts(frames_fts, rowid, name, digest_text, inputs, outputs)
VALUES ('delete', old.rowid, old.name, old.digest_text, old.inputs, old.outputs);
INSERT INTO frames_fts(rowid, name, digest_text, inputs, outputs)
VALUES (new.rowid, new.name, new.digest_text, new.inputs, new.outputs);
END;
`);
this.migrateToFts();
this.ftsEnabled = true;
logger.info("FTS5 full-text search initialized");
} catch (e) {
logger.warn(
"FTS5 initialization failed, falling back to LIKE search",
e
);
this.ftsEnabled = false;
}
}
/**
* One-time migration: populate FTS index from existing frames data
*/
migrateToFts() {
if (!this.db) return;
const version = this.db.prepare("SELECT MAX(version) as version FROM schema_version").get()?.version || 1;
if (version < 2) {
this.db.exec(`
INSERT OR IGNORE INTO frames_fts(rowid, name, digest_text, inputs, outputs)
SELECT rowid, name, digest_text, inputs, outputs FROM frames;
`);
this.db.prepare("INSERT OR REPLACE INTO schema_version (version) VALUES (?)").run(2);
logger.info(
"FTS5 index populated from existing frames (migration v1\u2192v2)"
);
}
if (version < 3) {
this.db.prepare("INSERT OR REPLACE INTO schema_version (version) VALUES (?)").run(3);
logger.info("Schema migration v2\u2192v3: GC importance_score support");
}
}
/**
* Initialize sqlite-vec for vector search
*/
initializeVec() {
if (!this.db || !this.embeddingProvider) return;
try {
let sqliteVec;
try {
sqliteVec = require("sqlite-vec");
} catch {
logger.info("sqlite-vec not installed, vector search disabled");
return;
}
sqliteVec.load(this.db);
this.db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS frame_embeddings USING vec0(
frame_id TEXT PRIMARY KEY,
embedding float[${this.embeddingDimension}]
);
`);
this.vecEnabled = true;
logger.info("sqlite-vec vector search initialized", {
dimension: this.embeddingDimension
});
} catch (e) {
logger.warn(
"sqlite-vec initialization failed, vector search disabled",
e
);
this.vecEnabled = false;
}
}
/**
* Rebuild the FTS5 index (for maintenance)
*/
async rebuildFtsIndex() {
if (!this.db) {
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
}
if (!this.ftsEnabled) {
logger.warn("FTS not enabled, skipping rebuild");
return;
}
this.db.exec("INSERT INTO frames_fts(frames_fts) VALUES('rebuild')");
logger.info("FTS5 index rebuilt");
}
/**
* Incremental garbage collection with generational compression.
*
* Phase 1 — Compress: Apply generational strategies to mature/old frames.
* - 'digest_only': Strip inputs/outputs/events, keep digest + anchors
* - 'anchors_only': Strip inputs/outputs/events/digest_json, keep digest_text + anchors
* - 'keep_all': No compression
*
* Phase 2 — Delete: Remove frames past retention cutoffs.
* - 'keep_forever': never deleted
* - 'default'/'archive': deleted after retentionDays (default 90)
* - 'ttl_30d': deleted after 30 days
* - 'ttl_7d': deleted after 7 days
*/
async runGC(options = {}) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const retentionDays = options.retentionDays ?? 90;
const batchSize = options.batchSize ?? 100;
const dryRun = options.dryRun ?? false;
const protectedRunIds = options.protectedRunIds ?? [];
const nowSec = Math.floor(Date.now() / 1e3);
const defaultCutoff = nowSec - retentionDays * 86400;
const ttl30dCutoff = nowSec - 30 * 86400;
const ttl7dCutoff = nowSec - 7 * 86400;
let framesCompressed = 0;
const gc = options.generationalGc;
if (gc && !dryRun) {
const youngDays = gc.youngCutoffDays ?? 1;
const matureDays = gc.matureCutoffDays ?? 7;
const oldDays = gc.oldCutoffDays ?? 30;
const youngCutoff = nowSec - youngDays * 86400;
const matureCutoff = nowSec - matureDays * 86400;
const matureStrategy = gc.mature_strategy ?? "digest_only";
if (matureStrategy !== "keep_all") {
const matureFrames = this.db.prepare(
`SELECT frame_id FROM frames
WHERE created_at < ? AND created_at >= ?
AND state = 'closed'
AND retention_policy != 'keep_forever'
AND inputs != '{}'
AND run_id NOT IN (SELECT value FROM json_each(?))
LIMIT ?`
).all(
youngCutoff,
matureCutoff,
JSON.stringify(protectedRunIds),
batchSize
);
if (matureFrames.length > 0) {
framesCompressed += this.compressFrames(
matureFrames.map((f) => f.frame_id),
matureStrategy
);
}
}
const oldStrategy = gc.old_strategy ?? "anchors_only";
if (oldStrategy !== "keep_all") {
const oldCutoff = nowSec - oldDays * 86400;
const oldFrames = this.db.prepare(
`SELECT frame_id FROM frames
WHERE created_at < ? AND created_at >= ?
AND state = 'closed'
AND retention_policy != 'keep_forever'
AND inputs != '{}'
AND run_id NOT IN (SELECT value FROM json_each(?))
LIMIT ?`
).all(
matureCutoff,
oldCutoff,
JSON.stringify(protectedRunIds),
batchSize
);
if (oldFrames.length > 0) {
framesCompressed += this.compressFrames(
oldFrames.map((f) => f.frame_id),
oldStrategy
);
}
}
}
const candidates = this.db.prepare(
`SELECT frame_id FROM frames
WHERE (
(retention_policy IN ('default', 'archive') AND created_at < ?)
OR (retention_policy = 'ttl_30d' AND created_at < ?)
OR (retention_policy = 'ttl_7d' AND created_at < ?)
)
AND retention_policy != 'keep_forever'
AND state = 'closed'
AND run_id NOT IN (SELECT value FROM json_each(?))
ORDER BY importance_score ASC, created_at ASC
LIMIT ?`
).all(
defaultCutoff,
ttl30dCutoff,
ttl7dCutoff,
JSON.stringify(protectedRunIds),
batchSize
);
const frameIds = candidates.map((r) => r.frame_id);
if (frameIds.length === 0) {
return {
framesDeleted: 0,
eventsDeleted: 0,
anchorsDeleted: 0,
embeddingsDeleted: 0,
ftsEntriesDeleted: 0,
framesCompressed
};
}
if (dryRun) {
const placeholders2 = frameIds.map(() => "?").join(",");
const eventsCount = this.db.prepare(
`SELECT COUNT(*) as count FROM events WHERE frame_id IN (${placeholders2})`
).get(...frameIds).count;
const anchorsCount = this.db.prepare(
`SELECT COUNT(*) as count FROM anchors WHERE frame_id IN (${placeholders2})`
).get(...frameIds).count;
let embeddingsCount = 0;
if (this.vecEnabled) {
embeddingsCount = this.db.prepare(
`SELECT COUNT(*) as count FROM frame_embeddings WHERE frame_id IN (${placeholders2})`
).get(...frameIds).count;
}
return {
framesDeleted: frameIds.length,
eventsDeleted: eventsCount,
anchorsDeleted: anchorsCount,
embeddingsDeleted: embeddingsCount,
ftsEntriesDeleted: frameIds.length,
// FTS has one entry per frame
framesCompressed
};
}
const placeholders = frameIds.map(() => "?").join(",");
let eventsDeleted = 0;
let anchorsDeleted = 0;
let embeddingsDeleted = 0;
this.db.prepare("BEGIN").run();
try {
if (this.vecEnabled) {
const embResult = this.db.prepare(
`DELETE FROM frame_embeddings WHERE frame_id IN (${placeholders})`
).run(...frameIds);
embeddingsDeleted = embResult.changes;
}
const evtResult = this.db.prepare(`DELETE FROM events WHERE frame_id IN (${placeholders})`).run(...frameIds);
eventsDeleted = evtResult.changes;
const ancResult = this.db.prepare(`DELETE FROM anchors WHERE frame_id IN (${placeholders})`).run(...frameIds);
anchorsDeleted = ancResult.changes;
this.db.prepare(`DELETE FROM frames WHERE frame_id IN (${placeholders})`).run(...frameIds);
this.db.prepare("COMMIT").run();
} catch (error) {
this.db.prepare("ROLLBACK").run();
throw error;
}
logger.info("GC completed", {
framesDeleted: frameIds.length,
eventsDeleted,
anchorsDeleted,
embeddingsDeleted,
framesCompressed
});
return {
framesDeleted: frameIds.length,
eventsDeleted,
anchorsDeleted,
embeddingsDeleted,
ftsEntriesDeleted: frameIds.length,
framesCompressed
};
}
/**
* Compute structural salience for a frame (range 0.3-1.0).
* Based on anchors, events, digest, and children.
*/
computeSalience(frameId, digestText) {
if (!this.db) return 0.3;
let score = 0.3;
const decisionCount = this.db.prepare(
"SELECT COUNT(*) as count FROM anchors WHERE frame_id = ? AND type = 'DECISION'"
).get(frameId).count;
if (decisionCount > 0) score += 0.15;
const eventCount = this.db.prepare("SELECT COUNT(*) as count FROM events WHERE frame_id = ?").get(frameId).count;
if (eventCount > 3) score += 0.1;
if (digestText) score += 0.15;
const childCount = this.db.prepare(
"SELECT COUNT(*) as count FROM frames WHERE parent_frame_id = ?"
).get(frameId).count;
if (childCount > 0) score += 0.1;
return Math.min(score, 1);
}
/**
* Compute importance score using Ebbinghaus retention decay + reinforcement.
*
* R(frame, t) = I_salience * e^(-lambda * dt) + sigma * sum(1 / (t - t_access_i))
*
* Score range: [0.05, 1.0] — never fully zero.
*/
computeImportanceScore(frameId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const frame = this.db.prepare(
"SELECT frame_id, digest_text, created_at FROM frames WHERE frame_id = ?"
).get(frameId);
if (!frame) return 0.3;
const salience = this.computeSalience(frameId, frame.digest_text);
const nowSec = Math.floor(Date.now() / 1e3);
const dtDays = Math.max(0, (nowSec - frame.created_at) / 86400);
const lambda = 0.05;
const decayTerm = salience * Math.exp(-lambda * dtDays);
const sigma = 0.1;
const accesses = this.db.prepare("SELECT accessed_at FROM frame_access_log WHERE frame_id = ?").all(frameId);
let reinforcement = 0;
for (const a of accesses) {
const gap = nowSec - a.accessed_at;
if (gap > 0) reinforcement += 1 / gap;
}
const raw = decayTerm + sigma * reinforcement;
return Math.round(Math.min(Math.max(raw, 0.05), 1) * 100) / 100;
}
/**
* Record a frame access for retention decay scoring.
* Inserts into frame_access_log and bumps access_count/last_accessed.
*/
recordFrameAccess(frameId) {
if (!this.db) return;
const nowSec = Math.floor(Date.now() / 1e3);
try {
this.db.prepare(
"INSERT INTO frame_access_log (frame_id, accessed_at) VALUES (?, ?)"
).run(frameId, nowSec);
this.db.prepare(
"UPDATE frames SET access_count = COALESCE(access_count, 0) + 1, last_accessed = ? WHERE frame_id = ?"
).run(nowSec, frameId);
} catch {
}
}
/**
* Compress a frame by stripping data according to strategy.
*
* - 'digest_only': Remove inputs, outputs, events. Keep digest_text, digest_json, anchors.
* - 'anchors_only': Remove inputs, outputs, events, digest_json. Keep digest_text, anchors.
*
* Returns true if the frame was compressed, false if not found.
*/
compressFrame(frameId, strategy) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const frame = this.db.prepare("SELECT frame_id FROM frames WHERE frame_id = ?").get(frameId);
if (!frame) return false;
this.db.prepare("BEGIN").run();
try {
if (strategy === "digest_only") {
this.db.prepare(
"UPDATE frames SET inputs = '{}', outputs = '{}' WHERE frame_id = ?"
).run(frameId);
} else {
this.db.prepare(
"UPDATE frames SET inputs = '{}', outputs = '{}', digest_json = '{}' WHERE frame_id = ?"
).run(frameId);
}
this.db.prepare("DELETE FROM events WHERE frame_id = ?").run(frameId);
if (this.vecEnabled) {
this.db.prepare("DELETE FROM frame_embeddings WHERE frame_id = ?").run(frameId);
}
this.db.prepare("COMMIT").run();
return true;
} catch (error) {
this.db.prepare("ROLLBACK").run();
throw error;
}
}
/**
* Compress multiple frames in a single transaction.
* Returns count of frames compressed.
*/
compressFrames(frameIds, strategy) {
if (!this.db || frameIds.length === 0) return 0;
const placeholders = frameIds.map(() => "?").join(",");
this.db.prepare("BEGIN").run();
try {
if (strategy === "digest_only") {
this.db.prepare(
`UPDATE frames SET inputs = '{}', outputs = '{}' WHERE frame_id IN (${placeholders})`
).run(...frameIds);
} else {
this.db.prepare(
`UPDATE frames SET inputs = '{}', outputs = '{}', digest_json = '{}' WHERE frame_id IN (${placeholders})`
).run(...frameIds);
}
this.db.prepare(`DELETE FROM events WHERE frame_id IN (${placeholders})`).run(...frameIds);
if (this.vecEnabled) {
this.db.prepare(
`DELETE FROM frame_embeddings WHERE frame_id IN (${placeholders})`
).run(...frameIds);
}
this.db.prepare("COMMIT").run();
return frameIds.length;
} catch (error) {
this.db.prepare("ROLLBACK").run();
throw error;
}
}
/**
* Get database file size in bytes.
*/
getDatabaseSize() {
if (!this.db) return 0;
const result = this.db.prepare(
"SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()"
).get();
return result?.size ?? 0;
}
/**
* Recompute importance scores for frames still at default score (0.5).
* Processes oldest frames first in batches.
* Returns count of frames updated.
*/
recomputeImportanceScores(batchSize = 100) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const frames = this.db.prepare(
"SELECT frame_id FROM frames WHERE importance_score = 0.5 ORDER BY created_at ASC LIMIT ?"
).all(batchSize);
const updateStmt = this.db.prepare(
"UPDATE frames SET importance_score = ? WHERE frame_id = ?"
);
let updated = 0;
for (const { frame_id } of frames) {
const score = this.computeImportanceScore(frame_id);
if (score !== 0.5) {
updateStmt.run(score, frame_id);
updated++;
}
}
if (updated > 0) {
logger.info("Recomputed importance scores", {
checked: frames.length,
updated
});
}
return updated;
}
async migrateSchema(targetVersion) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const currentVersion = await this.getSchemaVersion();
if (currentVersion >= targetVersion) {
logger.info("Schema already at target version", {
currentVersion,
targetVersion
});
return;
}
for (let v = currentVersion + 1; v <= targetVersion; v++) {
logger.info(`Applying migration to version ${v}`);
this.db.prepare("UPDATE schema_version SET version = ?").run(v);
}
}
async getSchemaVersion() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
try {
const result = this.db.prepare("SELECT MAX(version) as version FROM schema_version").get();
return result?.version || 0;
} catch (error) {
logger.debug("Schema version table not found, returning 0", {
error: error instanceof Error ? error.message : String(error)
});
return 0;
}
}
// Frame operations
async createFrame(frame) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const frameId = frame.frame_id || this.generateId();
this.db.prepare(
`
INSERT INTO frames (
frame_id, run_id, project_id, parent_frame_id, depth,
type, name, state, inputs, outputs, digest_text, digest_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
).run(
frameId,
frame.run_id,
frame.project_id || this.projectId,
frame.parent_frame_id || null,
frame.depth || 0,
frame.type,
frame.name,
frame.state || "active",
JSON.stringify(frame.inputs || {}),
JSON.stringify(frame.outputs || {}),
frame.digest_text || null,
JSON.stringify(frame.digest_json || {})
);
return frameId;
}
async getFrame(frameId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const row = this.db.prepare("SELECT * FROM frames WHERE frame_id = ?").get(frameId);
if (!row) return null;
return {
...row,
inputs: JSON.parse(row.inputs || "{}"),
outputs: JSON.parse(row.outputs || "{}"),
digest_json: JSON.parse(row.digest_json || "{}")
};
}
async updateFrame(frameId, updates) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const fields = [];
const values = [];
if (updates.state !== void 0) {
fields.push("state = ?");
values.push(updates.state);
}
if (updates.outputs !== void 0) {
fields.push("outputs = ?");
values.push(JSON.stringify(updates.outputs));
}
if (updates.digest_text !== void 0) {
fields.push("digest_text = ?");
values.push(updates.digest_text);
}
if (updates.digest_json !== void 0) {
fields.push("digest_json = ?");
values.push(JSON.stringify(updates.digest_json));
}
if (updates.closed_at !== void 0) {
fields.push("closed_at = ?");
values.push(updates.closed_at);
}
if (fields.length === 0) return;
values.push(frameId);
this.db.prepare(
`
UPDATE frames SET ${fields.join(", ")} WHERE frame_id = ?
`
).run(...values);
}
async deleteFrame(frameId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
await this.deleteFrameAnchors(frameId);
await this.deleteFrameEvents(frameId);
this.db.prepare("DELETE FROM frames WHERE frame_id = ?").run(frameId);
}
async getActiveFrames(runId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
let query = "SELECT * FROM frames WHERE state = 'active'";
const params = [];
if (runId) {
query += " AND run_id = ?";
params.push(runId);
}
query += " ORDER BY depth ASC, created_at ASC";
const rows = this.db.prepare(query).all(...params);
return rows.map((row) => ({
...row,
inputs: JSON.parse(row.inputs || "{}"),
outputs: JSON.parse(row.outputs || "{}"),
digest_json: JSON.parse(row.digest_json || "{}")
}));
}
async closeFrame(frameId, outputs) {
await this.updateFrame(frameId, {
state: "closed",
outputs,
closed_at: Date.now()
});
}
// Event operations
async createEvent(event) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const eventId = event.event_id || this.generateId();
this.db.prepare(
`
INSERT INTO events (event_id, run_id, frame_id, seq, event_type, payload, ts)
VALUES (?, ?, ?, ?, ?, ?, ?)
`
).run(
eventId,
event.run_id,
event.frame_id,
event.seq || 0,
event.event_type,
JSON.stringify(event.payload || {}),
event.ts || Date.now()
);
return eventId;
}
async getFrameEvents(frameId, options) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
let query = "SELECT * FROM events WHERE frame_id = ?";
query += this.buildOrderByClause(
options?.orderBy || "seq",
options?.orderDirection
);
query += this.buildLimitClause(options?.limit, options?.offset);
const rows = this.db.prepare(query).all(frameId);
return rows.map((row) => ({
...row,
payload: JSON.parse(row.payload || "{}")
}));
}
async deleteFrameEvents(frameId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.prepare("DELETE FROM events WHERE frame_id = ?").run(frameId);
}
// Anchor operations
async createAnchor(anchor) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const anchorId = anchor.anchor_id || this.generateId();
this.db.prepare(
`
INSERT INTO anchors (anchor_id, frame_id, project_id, type, text, priority, metadata)
VALUES (?, ?, ?, ?, ?, ?, ?)
`
).run(
anchorId,
anchor.frame_id,
anchor.project_id || this.projectId,
anchor.type,
anchor.text,
anchor.priority || 0,
JSON.stringify(anchor.metadata || {})
);
return anchorId;
}
async getFrameAnchors(frameId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const rows = this.db.prepare(
`
SELECT * FROM anchors WHERE frame_id = ?
ORDER BY priority DESC, created_at ASC
`
).all(frameId);
return rows.map((row) => ({
...row,
metadata: JSON.parse(row.metadata || "{}")
}));
}
async deleteFrameAnchors(frameId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.prepare("DELETE FROM anchors WHERE frame_id = ?").run(frameId);
}
// Full-text search with FTS5 + BM25 ranking (fallback to LIKE)
async search(options) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
if (this.ftsEnabled && options.query.trim()) {
try {
return this.searchFts(options);
} catch (e) {
logger.debug("FTS search failed, falling back to LIKE", {
error: e instanceof Error ? e.message : String(e),
query: options.query
});
}
}
return this.searchLike(options);
}
/**
* Sanitize user input for FTS5 MATCH queries.
* - Strips FTS5 operators and special syntax
* - Wraps individual terms in double quotes for exact matching
* - Joins with implicit AND
* - Supports prefix matching when original query ends with *
*/
sanitizeFtsQuery(query) {
const wantsPrefix = query.trimEnd().endsWith("*");
const cleaned = query.replace(/['"(){}[\]^~*\\,]/g, " ").replace(/\b(AND|OR|NOT|NEAR)\b/gi, "").trim();
const terms = cleaned.split(/\s+/).filter((t) => t.length > 0);
if (terms.length === 0) return '""';
const quoted = terms.map((t) => `"${t}"`);
if (wantsPrefix) {
quoted[quoted.length - 1] = quoted[quoted.length - 1] + "*";
}
return quoted.join(" ");
}
/**
* FTS5 MATCH search with BM25 ranking
*/
searchFts(options) {
const sanitizedQuery = this.sanitizeFtsQuery(options.query);
const boost = options.boost || {};
const w0 = boost["name"] || 10;
const w1 = boost["digest_text"] || 5;
const w2 = boost["inputs"] || 2;
const w3 = boost["outputs"] || 1;
const projectFilter = options.projectId ? "AND f.project_id = ?" : "";
const sql = `
SELECT f.*, -bm25(frames_fts, ${w0}, ${w1}, ${w2}, ${w3}) as score
FROM frames_fts fts
JOIN frames f ON f.rowid = fts.rowid
WHERE frames_fts MATCH ?
${projectFilter}
ORDER BY score DESC
LIMIT ? OFFSET ?
`;
const limit = options.limit || 50;
const offset = options.offset || 0;
const params = [sanitizedQuery];
if (options.projectId) params.push(options.projectId);
params.push(limit, offset);
const rows = this.db.prepare(sql).all(...params);
return rows.map((row) => ({
...row,
inputs: JSON.parse(row.inputs || "{}"),
outputs: JSON.parse(row.outputs || "{}"),
digest_json: JSON.parse(row.digest_json || "{}")
}));
}
/**
* Fallback LIKE search for when FTS is unavailable
*/
searchLike(options) {
const projectFilter = options.projectId ? "AND project_id = ?" : "";
const sql = `
SELECT *,
CASE
WHEN name LIKE ? THEN 1.0
WHEN digest_text LIKE ? THEN 0.8
WHEN inputs LIKE ? THEN 0.6
ELSE 0.5
END as score
FROM frames
WHERE (name LIKE ? OR digest_text LIKE ? OR inputs LIKE ?)
${projectFilter}
ORDER BY score DESC
`;
const likeParam = `%${options.query}%`;
const params = Array(6).fill(likeParam);
if (options.projectId) params.push(options.projectId);
let rows = this.db.prepare(sql).all(...params);
if (options.scoreThreshold) {
rows = rows.filter((row) => row.score >= options.scoreThreshold);
}
if (options.limit || options.offset) {
const start = options.offset || 0;
const end = options.limit ? start + options.limit : rows.length;
rows = rows.slice(start, end);
}
return rows.map((row) => ({
...row,
inputs: JSON.parse(row.inputs || "{}"),
outputs: JSON.parse(row.outputs || "{}"),
digest_json: JSON.parse(row.digest_json || "{}")
}));
}
async searchByVector(embedding, options) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
if (!this.vecEnabled) {
logger.warn("Vector search not available (sqlite-vec not loaded)");
return [];
}
const limit = options?.limit || 20;
const sql = `
SELECT f.*, ve.distance as similarity
FROM frame_embeddings ve
JOIN frames f ON f.frame_id = ve.frame_id
WHERE ve.embedding MATCH ?
ORDER BY ve.distance
LIMIT ?
`;
const rows = this.db.prepare(sql).all(JSON.stringify(embedding), limit);
return rows.map((row) => ({
...row,
inputs: JSON.parse(row.inputs || "{}"),
outputs: JSON.parse(row.outputs || "{}"),
digest_json: JSON.parse(row.digest_json || "{}")
}));
}
async searchHybrid(textQuery, embedding, weights, mergeStrategy) {
const textResults = await this.search({ query: textQuery, limit: 50 });
const vecResults = this.vecEnabled ? await this.searchByVector(embedding, { limit: 50 }) : [];
if (vecResults.length === 0) {
return textResults;
}
if (textResults.length === 0) {
return this.normalizeVectorOnly(vecResults);
}
if (mergeStrategy === "rrf") {
return this.mergeByRRF(textResults, vecResults);
}
return this.mergeByWeightedScore(textResults, vecResults, weights);
}
/**
* Merge text and vector results using min-max normalized weighted scoring.
* Both score types are scaled to [0, 1] before combining.
*/
mergeByWeightedScore(textResults, vecResults, weights) {
const textWeight = weights?.text ?? 0.6;
const vecWeight = weights?.vector ?? 0.4;
const scoreMap = /* @__PURE__ */ new Map();
const textScores = textResults.map((r) => r.score);
const minText = Math.min(...textScores);
const maxText = Math.max(...textScores);
const rangeText = maxText - minText;
for (const r of textResults) {
const normalized = rangeText === 0 ? 1 : (r.score - minText) / rangeText;
scoreMap.set(r.frame_id, { frame: r, score: normalized * textWeight });
}
const distances = vecResults.map((r) => r.similarity);
const minDist = Math.min(...distances);
const maxDist = Math.max(...distances);
const rangeDist = maxDist - minDist;
for (const r of vecResults) {
const normalized = rangeDist === 0 ? 1 : 1 - (r.similarity - minDist) / rangeDist;
const existing = scoreMap.get(r.frame_id);
if (existing) {
existing.score += normalized * vecWeight;
} else {
scoreMap.set(r.frame_id, { frame: r, score: normalized * vecWeight });
}
}
return Array.from(scoreMap.values()).sort((a, b) => b.score - a.score).map(({ frame, score }) => ({ ...frame, score }));
}
/**
* Merge text and vector results using Reciprocal Rank Fusion.
* Rank-based merging that is immune to score scale differences.
*/
mergeByRRF(textResults, vecResults, k = 60) {
const scoreMap = /* @__PURE__ */ new Map();
for (let rank = 0; rank < textResults.length; rank++) {
const r = textResults[rank];
const rrfScore = 1 / (k + rank + 1);
scoreMap.set(r.frame_id, { frame: r, score: rrfScore });
}
for (let rank = 0; rank < vecResults.length; rank++) {
const r = vecResults[rank];
const rrfScore = 1 / (k + rank + 1);
const existing = scoreMap.get(r.frame_id);
if (existing) {
existing.score += rrfScore;
} else {
scoreMap.set(r.frame_id, { frame: r, score: rrfScore });
}
}
return Array.from(scoreMap.values()).sort((a, b) => b.score - a.score).map(({ frame, score }) => ({ ...frame, score }));
}
/**
* Convert vector-only results (distances) to [0, 1] scores.
*/
normalizeVectorOnly(vecResults) {
if (vecResults.length === 0) return [];
const distances = vecResults.map((r) => r.similarity);
const minDist = Math.min(...distances);
const maxDist = Math.max(...distances);
const range = maxDist - minDist;
return vecResults.map((r) => ({
...r,
score: range === 0 ? 1 : 1 - (r.similarity - minDist) / range
}));
}
/**
* Store an embedding for a frame
*/
async storeEmbedding(frameId, embedding) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
if (!this.vecEnabled) return;
this.db.prepare(
"INSERT OR REPLACE INTO frame_embeddings (frame_id, embedding) VALUES (?, ?)"
).run(frameId, JSON.stringify(embedding));
}
/**
* Get a maintenance state value by key
*/
async getMaintenanceState(key) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const row = this.db.prepare("SELECT value FROM maintenance_state WHERE key = ?").get(key);
return row?.value ?? null;
}
/**
* Set a maintenance state value by key
*/
async setMaintenanceState(key, value) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.prepare(
"INSERT OR REPLACE INTO maintenance_state (key, value, updated_at) VALUES (?, ?, ?)"
).run(key, value, Date.now());
}
/**
* Get frames that are missing embeddings
*/
async getFramesMissingEmbeddings(limit = 50, sinceRowid) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const rowidFilter = sinceRowid != null ? "AND f.rowid > ?" : "";
const sql = `
SELECT f.* FROM frames f
LEFT JOIN frame_embeddings ve ON f.frame_id = ve.frame_id
WHERE ve.frame_id IS NULL ${rowidFilter}
ORDER BY f.rowid ASC
LIMIT ?
`;
const params = [];
if (sinceRowid != null) params.push(sinceRowid);
params.push(limit);
const rows = this.db.prepare(sql).all(...params);
return rows.map((row) => ({
...row,
inputs: JSON.parse(row.inputs || "{}"),
outputs: JSON.parse(row.outputs || "{}"),
digest_json: JSON.parse(row.digest_json || "{}")
}));
}
// Project registry operations
async registerProject(project) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.prepare(
`INSERT OR REPLACE INTO project_registry
(project_id, repo_path, display_name, db_path, is_active, created_at, last_accessed)
VALUES (?, ?, ?, ?, 0, ?, ?)`
).run(
project.projectId,
project.repoPath,
project.displayName || null,
project.dbPath,
Date.now(),
Date.now()
);
}
async getRegisteredProjects() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const rows = this.db.prepare("SELECT * FROM project_registry ORDER BY last_accessed DESC").all();
return rows.map((row) => ({
projectId: row.project_id,
repoPath: row.repo_path,
displayName: row.display_name,
dbPath: row.db_path,
isActive: row.is_active === 1,
createdAt: row.created_at,
lastAccessed: row.last_accessed
}));
}
async setActiveProject(projectId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.prepare("UPDATE project_registry SET is_active = 0").run();
this.db.prepare(
"UPDATE project_registry SET is_active = 1, last_accessed = ? WHERE project_id = ?"
).run(Date.now(), projectId);
}
async getActiveProject() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const row = this.db.prepare(
"SELECT project_id FROM project_registry WHERE is_active = 1 LIMIT 1"
).get();
return row?.project_id ?? null;
}
async removeProject(projectId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const result = this.db.prepare("DELETE FROM project_registry WHERE project_id = ?").run(projectId);
return result.changes > 0;
}
async touchProject(projectId) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.prepare(
"UPDATE project_registry SET last_accessed = ? WHERE project_id = ?"
).run(Date.now(), projectId);
}
// Basic aggregation
async aggregate(table, options) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const metrics = options.metrics.map(
(m) => `${m.operation}(${m.field}) AS ${m.alias || `${m.operation}_${m.field}`}`
).join(", ");
let sql = `SELECT ${options.groupBy.join(", ")}, ${metrics} FROM ${table}`;
sql += ` GROUP BY ${options.groupBy.join(", ")}`;
if (options.having) {
const havingClauses = Object.entries(options.having).map(
([key, value]) => `${key} ${typeof value === "object" ? value.op : "="} ?`
);
sql += ` HAVING ${havingClauses.join(" AND ")}`;
}
return this.db.prepare(sql).all(...Object.values(options.having || {}));
}
// Pattern detection (basic)
async detectPatterns(timeRange) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
let sql = `
SELECT type as pattern, type, COUNT(*) as frequency, MAX(created_at) as last_seen
FROM frames
`;
const params = [];
if (timeRange) {
sql += " WHERE created_at >= ? AND created_at <= ?";
params.push(
Math.floor(timeRange.start.getTime() / 1e3),
Math.floor(timeRange.end.getTime() / 1e3)
);
}
sql += " GROUP BY type HAVING COUNT(*) > 1 ORDER BY frequency DESC";
const rows = this.db.prepare(sql).all(...params);
return rows.map((row) => ({
pattern: row.pattern,
type: row.type,
frequency: row.frequency,
lastSeen: new Date(row.last_seen * 1e3)
}));
}
// Retrieval logging
async logRetrieval(entry) {
if (!this.db) return;
try {
this.db.prepare(
`INSERT INTO retrieval_log (query_text, strategy, results_count, top_score, latency_ms, result_frame_ids, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`
).run(
entry.queryText,
entry.strategy,
entry.resultsCount,
entry.topScore,
entry.latencyMs,
JSON.stringify(entry.resultFrameIds),
Date.now()
);
} catch (e) {
logger.warn("Failed to log retrieval", e);
}
}
async getRetrievalStats(sinceDays) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const sinceMs = sinceDays ? Date.now() - sinceDays * 24 * 60 * 60 * 1e3 : 0;
const whereClause = sinceMs ? "WHERE created_at >= ?" : "";
const params = sinceMs ? [sinceMs] : [];
const agg = this.db.prepare(
`SELECT
COUNT(*) as total_queries,
COALESCE(AVG(latency_ms), 0) as avg_latency_ms,
COALESCE(AVG(results_count), 0) as avg_results_count,
COUNT(CASE WHEN results_count = 0 THEN 1 END) as queries_with_no_results
FROM retrieval_log ${whereClause}`
).get(...params);
const p95Offset = Math.max(0, Math.round(agg.total_queries * 0.95) - 1);
const p95Row = agg.total_queries > 0 ? this.db.prepare(
`SELECT latency_ms FROM retrieval_log ${whereClause}
ORDER BY latency_ms ASC
LIMIT 1 OFFSET ?`
).get(...params, p95Offset) : void 0;
const stratRows = this.db.prepare(
`SELECT strategy, COUNT(*) as count FROM retrieval_log ${whereClause}
GROUP BY strategy`
).all(...params);
const strategyDistribution = {};
for (const row of stratRows) {
strategyDistribution[row.strategy] = row.count;
}
return {
totalQueries: agg.total_queries,
avgLatencyMs: Math.round(agg.avg_latency_ms * 100) / 100,
p95LatencyMs: p95Row?.latency_ms ?? 0,
strategyDistribution,
avgResultsCount: Math.round(agg.avg_results_count * 100) / 100,
queriesWithNoResults: agg.queries_with_no_results
};
}
// Bulk operations
async executeBulk(operations) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
await this.inTransaction(async () => {
for (const op of operations) {
switch (op.type) {
case "insert":
const insertCols = Object.keys(op.data);
const insertPlaceholders = insertCols.map(() => "?").join(",");
this.db.prepare(
`INSERT INTO ${op.table} (${insertCols.join(",")}) VALUES (${insertPlaceholders})`
).run(...Object.values(op.data));
break;
case "update":
const updateSets = Object.keys(op.data).map((k) => `${k} = ?`).join(",");
const whereClause = this.buildWhereClause(op.where || {});
this.db.prepare(
`UPDATE ${op.table} SET ${updateSets} ${whereClause}`
).run(...Object.values(op.data), ...Object.values(op.where || {}));
break;
case "delete":
const deleteWhere = this.buildWhereClause(op.where || {});
this.db.prepare(`DELETE FROM ${op.table} ${deleteWhere}`).run(
...Object.values(op.where || {})
);
break;
}
}
});
}
async getTablePage(table, offset, limit) {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const safeLimit = Math.max(1, Math.min(limit, 1e4));
const safeOffset = Math.max(0, offset);
const allowedTables = ["frames", "events", "anchors"];
if (!allowedTables.includes(table)) {
throw new DatabaseError(
`Invalid table name: ${table}`,
ErrorCode.DB_QUERY_FAILED,
{ table }
);
}
return this.db.prepare(`SELECT * FROM ${table} ORDER BY rowid LIMIT ? OFFSET ?`).all(safeLimit, safeOffset);
}
async vacuum() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.pragma("vacuum");
logger.info("SQLite database vacuumed");
}
async analyze() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
this.db.pragma("analyze");
logger.info("SQLite database analyzed");
}
// Statistics
async getStats() {
if (!this.db)
throw new DatabaseError(
"Database not connected",
ErrorCode.DB_CONNECTION_FAILED
);
const stats = {
totalFrames: this.db.prepare("SELECT COUNT(*) as count FROM frames").get().count,
activeFrames: this.db.prepare(
"SELECT COUNT(*) as count FROM frames WHERE state = 'active'"
).get().count,
totalEvents: this.db.prepare("SELECT COUNT(*) as count FROM events").get().count,
totalAnchors: this.db.prepare("SELECT COUNT(*) as count FROM anchors").get().count,
diskUsage: 0
};
try {
const fileStats = await fs.stat(this.dbPath);
stats.diskUsage = fileStats.size;
} catch (error) {
logger.debug("Failed to get database file size", {
dbPath: this.dbPath,
error: error instanceof Error ? error.message : String(error)
});
}
return stats;
}
async getQueryStats() {
logger.warn("Query