agentic-qe
Version:
Agentic Quality Engineering Fleet System - AI-driven quality management platform
460 lines • 15.9 kB
JavaScript
"use strict";
/**
* MemoryManager - Intelligent memory management for AQE Fleet
*
* Provides in-memory storage with TTL support, namespacing, and optional
* SQLite persistence for cross-session memory and agent coordination.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryManager = void 0;
const events_1 = require("events");
const Database_1 = require("../utils/Database");
const Logger_1 = require("../utils/Logger");
class MemoryManager extends events_1.EventEmitter {
constructor(database) {
super();
this.storage = new Map();
this.defaultTTL = 3600000; // 1 hour in milliseconds
this.initialized = false;
this.database = database || new Database_1.Database();
this.logger = Logger_1.Logger.getInstance();
// Setup automatic cleanup every 5 minutes
this.cleanupInterval = setInterval(() => {
this.cleanupExpired();
}, 5 * 60 * 1000);
}
/**
* Initialize the memory manager
*/
async initialize() {
if (this.initialized) {
return;
}
try {
await this.database.initialize();
// Load persistent memory from database if available
await this.loadPersistentMemory();
this.initialized = true;
this.logger.info('MemoryManager initialized successfully');
}
catch (error) {
this.logger.error('Failed to initialize MemoryManager:', error);
throw error;
}
}
/**
* Store a value in memory
*/
async store(key, value, options = {}) {
const namespace = options.namespace || 'default';
const fullKey = this.createFullKey(key, namespace);
const ttl = options.ttl || this.defaultTTL;
const expiresAt = ttl > 0 ? Date.now() + ttl : undefined;
const record = {
key,
value,
namespace,
ttl,
metadata: options.metadata,
timestamp: Date.now()
};
// Store in memory
this.storage.set(fullKey, record);
// Store in database if persistence is enabled
if (options.persist && this.database) {
try {
// Use the database's run method to store the record
const sql = `INSERT OR REPLACE INTO memory_store (key, value, namespace, ttl, metadata, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)`;
await this.database.run(sql, [
key,
JSON.stringify(record),
namespace,
ttl,
JSON.stringify(options.metadata || {}),
expiresAt ? new Date(expiresAt).toISOString() : null
]);
}
catch (error) {
this.logger.warn('Failed to persist memory to database:', error);
}
}
this.emit('store', { key, namespace, value, ttl });
this.logger.debug(`Stored key ${fullKey}`, {
namespace,
ttl,
persist: options.persist
});
}
/**
* Retrieve a value from memory
*/
async retrieve(key, namespace = 'default') {
const fullKey = this.createFullKey(key, namespace);
const record = this.storage.get(fullKey);
if (!record) {
// Try loading from database if not in memory
const persistentRecord = await this.loadFromDatabase(key, namespace);
if (persistentRecord) {
// Add back to memory
this.storage.set(fullKey, persistentRecord);
return persistentRecord.value;
}
return undefined;
}
// Check if expired
if (this.isExpired(record)) {
this.storage.delete(fullKey);
this.emit('expired', { key, namespace });
return undefined;
}
this.emit('retrieve', { key, namespace, value: record.value });
return record.value;
}
/**
* Delete a key from memory
*/
async delete(key, namespace = 'default') {
const fullKey = this.createFullKey(key, namespace);
const existed = this.storage.delete(fullKey);
// Also delete from database
if (this.database) {
try {
const sql = `DELETE FROM memory_store WHERE key = ? AND namespace = ?`;
await this.database.run(sql, [key, namespace]);
}
catch (error) {
this.logger.warn('Failed to delete from database:', error);
}
}
if (existed) {
this.emit('delete', { key, namespace });
}
return existed;
}
/**
* Check if a key exists
*/
async exists(key, namespace = 'default') {
const fullKey = this.createFullKey(key, namespace);
const record = this.storage.get(fullKey);
if (!record) {
// Check database
const persistentRecord = await this.loadFromDatabase(key, namespace);
return !!persistentRecord;
}
// Check if expired
if (this.isExpired(record)) {
this.storage.delete(fullKey);
return false;
}
return true;
}
/**
* List all keys in a namespace
*/
async list(namespace = 'default') {
const prefix = `${namespace}:`;
const keys = [];
// Get from memory
for (const [fullKey, record] of this.storage.entries()) {
if (fullKey.startsWith(prefix) && !this.isExpired(record)) {
keys.push(record.key);
}
}
// Get from database
if (this.database) {
try {
const sql = `SELECT key FROM memory_store WHERE namespace = ? AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)`;
const rows = await this.database.all(sql, [namespace]);
const dbKeys = rows.map(row => row.key);
// Merge and deduplicate
const allKeys = new Set([...keys, ...dbKeys]);
return Array.from(allKeys);
}
catch (error) {
this.logger.warn('Failed to list keys from database:', error);
}
}
return keys;
}
/**
* Search for keys by pattern
*/
async search(options = {}) {
const { namespace = 'default', pattern = '.*', limit = 100, includeExpired = false } = options;
const regex = new RegExp(pattern, 'i');
const results = [];
const prefix = `${namespace}:`;
for (const [fullKey, record] of this.storage.entries()) {
if (results.length >= limit)
break;
if (fullKey.startsWith(prefix)) {
const isExpired = this.isExpired(record);
if (!includeExpired && isExpired) {
continue;
}
if (regex.test(record.key) || regex.test(JSON.stringify(record.value))) {
results.push({ ...record });
}
}
}
return results;
}
/**
* Clear all keys in a namespace
*/
async clear(namespace = 'default') {
const prefix = `${namespace}:`;
const keysToDelete = [];
// Find keys to delete
for (const fullKey of this.storage.keys()) {
if (fullKey.startsWith(prefix)) {
keysToDelete.push(fullKey);
}
}
// Delete from memory
for (const fullKey of keysToDelete) {
this.storage.delete(fullKey);
}
// Clear from database
if (this.database) {
try {
const sql = `DELETE FROM memory_store WHERE namespace = ?`;
await this.database.run(sql, [namespace]);
}
catch (error) {
this.logger.warn('Failed to clear namespace from database:', error);
}
}
this.emit('clear', { namespace, count: keysToDelete.length });
return keysToDelete.length;
}
/**
* Get memory statistics
*/
getStats() {
const namespaces = new Set();
let totalSize = 0;
let expiredKeys = 0;
let persistentKeys = 0;
for (const [, record] of this.storage.entries()) {
namespaces.add(record.namespace);
totalSize += JSON.stringify(record).length;
if (this.isExpired(record)) {
expiredKeys++;
}
if (record.ttl === undefined || record.ttl <= 0) {
persistentKeys++;
}
}
return {
totalKeys: this.storage.size,
totalSize,
namespaces: Array.from(namespaces),
expiredKeys,
persistentKeys
};
}
/**
* Set TTL for an existing key
*/
async setTTL(key, ttl, namespace = 'default') {
const fullKey = this.createFullKey(key, namespace);
const record = this.storage.get(fullKey);
if (!record) {
return false;
}
record.ttl = ttl;
record.timestamp = Date.now(); // Reset timestamp
this.storage.set(fullKey, record);
this.emit('ttl_updated', { key, namespace, ttl });
return true;
}
/**
* Get TTL for a key
*/
async getTTL(key, namespace = 'default') {
const fullKey = this.createFullKey(key, namespace);
const record = this.storage.get(fullKey);
if (!record) {
return undefined;
}
if (record.ttl === undefined || record.ttl <= 0) {
return -1; // Persistent key
}
const elapsed = Date.now() - record.timestamp;
const remaining = record.ttl - elapsed;
return remaining > 0 ? remaining : 0;
}
/**
* Export memory data for backup
*/
async export(namespace) {
const records = [];
for (const [, record] of this.storage.entries()) {
if (!namespace || record.namespace === namespace) {
if (!this.isExpired(record)) {
records.push({ ...record });
}
}
}
return records;
}
/**
* Import memory data from backup
*/
async import(records) {
let imported = 0;
for (const record of records) {
try {
await this.store(record.key, record.value, {
namespace: record.namespace,
ttl: record.ttl,
metadata: record.metadata,
persist: true
});
imported++;
}
catch (error) {
this.logger.warn(`Failed to import record ${record.key}:`, error);
}
}
this.logger.info(`Imported ${imported} memory records`);
return imported;
}
/**
* Cleanup expired keys
*/
cleanupExpired() {
const expiredKeys = [];
for (const [fullKey, record] of this.storage.entries()) {
if (this.isExpired(record)) {
expiredKeys.push(fullKey);
}
}
for (const fullKey of expiredKeys) {
this.storage.delete(fullKey);
}
if (expiredKeys.length > 0) {
this.emit('cleanup', { expiredCount: expiredKeys.length });
this.logger.debug(`Cleaned up ${expiredKeys.length} expired keys`);
}
}
/**
* Shutdown memory manager
*/
async shutdown() {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
}
// Save all non-expired data to database
await this.saveToPersistence();
await this.database.close();
this.storage.clear();
this.removeAllListeners();
this.initialized = false;
this.logger.info('MemoryManager shutdown complete');
}
/**
* Create full key with namespace
*/
createFullKey(key, namespace) {
return `${namespace}:${key}`;
}
/**
* Check if a record is expired
*/
isExpired(record) {
if (record.ttl === undefined || record.ttl <= 0) {
return false; // Persistent key
}
const elapsed = Date.now() - record.timestamp;
return elapsed > record.ttl;
}
/**
* Load a record from database
*/
async loadFromDatabase(key, namespace) {
if (!this.database) {
return undefined;
}
try {
const sql = `SELECT value FROM memory_store WHERE key = ? AND namespace = ? AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)`;
const row = await this.database.get(sql, [key, namespace]);
if (row) {
return JSON.parse(row.value);
}
}
catch (error) {
this.logger.warn('Failed to load from database:', error);
}
return undefined;
}
/**
* Load persistent memory from database
*/
async loadPersistentMemory() {
if (!this.database) {
return;
}
try {
// This is a simplified implementation
// In a real implementation, you would iterate through all namespaces
const namespaces = ['default', 'fleet', 'agents', 'tasks', 'coordination'];
for (const namespace of namespaces) {
try {
const sql = `SELECT key FROM memory_store WHERE namespace = ? AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)`;
const rows = await this.database.all(sql, [namespace]);
for (const row of rows) {
const record = await this.loadFromDatabase(row.key, namespace);
if (record && !this.isExpired(record)) {
const fullKey = this.createFullKey(row.key, namespace);
this.storage.set(fullKey, record);
}
}
}
catch (error) {
this.logger.warn(`Failed to load keys for namespace ${namespace}:`, error);
}
}
this.logger.info('Loaded persistent memory from database');
}
catch (error) {
this.logger.warn('Failed to load persistent memory:', error);
}
}
/**
* Save memory to persistence
*/
async saveToPersistence() {
if (!this.database) {
return;
}
try {
let saved = 0;
for (const [, record] of this.storage.entries()) {
if (!this.isExpired(record)) {
const sql = `INSERT OR REPLACE INTO memory_store (key, value, namespace, ttl, metadata, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?)`;
const expiresAt = record.ttl && record.ttl > 0 ? new Date(record.timestamp + record.ttl).toISOString() : null;
await this.database.run(sql, [
record.key,
JSON.stringify(record),
record.namespace,
record.ttl || 0,
JSON.stringify(record.metadata || {}),
expiresAt
]);
saved++;
}
}
this.logger.info(`Saved ${saved} memory records to persistence`);
}
catch (error) {
this.logger.warn('Failed to save to persistence:', error);
}
}
}
exports.MemoryManager = MemoryManager;
//# sourceMappingURL=MemoryManager.js.map