@dollhousemcp/mcp-server
Version:
DollhouseMCP - A Model Context Protocol (MCP) server that enables dynamic AI persona management from markdown files, allowing Claude and other compatible AI assistants to activate and switch between different behavioral personas.
1,045 lines (1,044 loc) • 270 kB
JavaScript
/**
* MemoryManager - Implementation of IElementManager for Memory elements
* Handles CRUD operations and lifecycle management for memories implementing IElement
*
* FIXES IMPLEMENTED:
* 1. CRITICAL: Fixed race conditions in file operations by using FileLockManager for atomic reads/writes
* 2. HIGH: Fixed unvalidated YAML parsing vulnerability by using SecureYamlParser
* 3. MEDIUM: All user inputs are now validated and sanitized
* 4. MEDIUM: Audit logging added for security operations
* 5. MEDIUM: Path validation prevents directory traversal attacks
*/
import { Memory } from './Memory.js';
import { ElementType } from '../../portfolio/types.js';
import { toSingularLabel } from '../../utils/elementTypeNormalization.js';
import { BaseElementManager } from '../base/BaseElementManager.js';
import { getValidatedScanCooldown, getValidatedIndexDebounce, STORAGE_LAYER_CONFIG } from '../../config/performance-constants.js';
import { MemoryStorageLayer } from '../../storage/MemoryStorageLayer.js';
import { MemoryMetadataExtractor } from '../../storage/MemoryMetadataExtractor.js';
import { LRUCache } from '../../cache/LRUCache.js';
import { SecurityMonitor } from '../../security/securityMonitor.js';
import { logger } from '../../utils/logger.js';
import { sanitizeInput } from '../../security/InputValidator.js';
import { ContentValidator } from '../../security/contentValidator.js';
import { SecureYamlParser } from '../../security/secureYamlParser.js';
import { SECURITY_LIMITS } from '../../security/constants.js';
import { MEMORY_CONSTANTS, MEMORY_SECURITY_EVENTS } from './constants.js';
import { MemoryType } from './types.js';
import * as path from 'path';
import * as crypto from 'crypto';
import { fileURLToPath } from 'url';
import { ElementMessages } from '../../utils/elementMessages.js';
import { sanitizeGatekeeperPolicy, getGatekeeperAuthoringErrors } from '../../handlers/mcp-aql/policies/ElementPolicies.js';
// Issue #83: Centralized active element limits (configurable via env vars)
import { getActiveElementLimitConfig, getMaxActiveLimit } from '../../config/active-element-limits.js';
/**
* Issue #13: Utility function to check if a filename is a backup file
* Backup files should not be loaded by list() - they are archived data, not active memories
*
* Backup patterns include:
* - .backup- timestamp pattern (e.g., name.backup-2025-11-14-22-40-57-303.yaml)
* - Any file with 'backup' in the name (case-insensitive)
* - Files in backup/ or backups/ directories (handled at directory level)
*
* @param filename - The filename to check (without directory path)
* @returns true if the file is a backup file that should be excluded
*/
function isBackupFile(filename) {
return filename.includes('.backup-') || filename.toLowerCase().includes('backup');
}
/**
* Issue #39: Check if a memory name contains backup patterns (corrupted name)
* This can happen when backup file metadata leaks into the canonical file
*
* Pattern: name.backup-YYYY-MM-DD-HH-mm-ss-SSS or name.backup-...-vN
*
* @param name - The memory name to check
* @returns true if the name contains backup patterns indicating corruption
*/
function isCorruptedBackupName(name) {
// Check for .backup- timestamp pattern in the name
return /\.backup-\d{4}-\d{2}-\d{2}/.test(name);
}
/**
* Issue #39: Extract the original name from a corrupted backup name
*
* Examples:
* - "dollhousemcp-baseline-knowledge.backup-2025-11-14-22-40-57-303"
* -> "dollhousemcp-baseline-knowledge"
* - "my-memory.backup-2025-01-01-00-00-00-000-v2"
* -> "my-memory"
*
* @param corruptedName - The corrupted name containing backup pattern
* @returns The original name with backup suffix removed
*/
function extractOriginalName(corruptedName) {
// Remove .backup-YYYY-MM-DD-HH-mm-ss-SSS and any -vN suffix
return corruptedName.replace(/\.backup-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-\d{3}(-v\d+)?$/, '');
}
export class MemoryManager extends BaseElementManager {
metadataService;
memoriesDir;
// Phase 2: Bounded content hash index replaces unbounded Map
contentHashIndex = new LRUCache({
name: 'memory:contentHash',
maxSize: 5000,
maxMemoryMB: 5,
ttlMs: 0,
});
contentHashByPath = new LRUCache({
name: 'memory:contentHashReverse',
maxSize: 5000,
maxMemoryMB: 5,
ttlMs: 0,
});
triggerValidationService;
validationService;
serializationService;
// Track active memories by name (stable identifier) - Issue #18 Phase 4
activeMemoryNames = new Set();
constructor(portfolioManager, fileLockManager, fileOperationsService, validationRegistry, serializationService, metadataService, fileWatchService, memoryBudget, backupService) {
super(ElementType.MEMORY, portfolioManager, fileLockManager, { fileWatchService, memoryBudget, backupService }, fileOperationsService, validationRegistry);
this.metadataService = metadataService;
this.memoriesDir = this.elementDir;
this.triggerValidationService = validationRegistry.getTriggerValidationService();
this.validationService = validationRegistry.getValidationService();
this.serializationService = serializationService;
}
/**
* Phase 2: Override factory to use MemoryStorageLayer for multi-directory scanning.
*/
createStorageLayer(fileOperationsService) {
// Note: Use this.elementDir (set by BaseElementManager before this is called),
// not this.memoriesDir (set after super() returns).
return new MemoryStorageLayer(fileOperationsService, {
memoriesDir: this.elementDir,
scanCooldownMs: getValidatedScanCooldown(),
indexDebounceMs: getValidatedIndexDebounce(),
fileFilter: (filename) => !isBackupFile(filename),
});
}
getElementLabel() {
return 'memory';
}
// MemoryManager has its own backup system (moveToUserBackup) and its
// save/delete don't call super — no-op the universal backup hooks.
async createBackupBeforeSave() { }
async createBackupBeforeDelete() { return false; }
/**
* Override clearCache to also clear memory-specific caches.
* Phase 2: Removed unbounded memoryCache and dateFoldersCache.
*/
clearCache() {
super.clearCache();
this.contentHashIndex.clear();
this.contentHashByPath.clear();
}
/**
* Override dispose to flush MemoryStorageLayer _index.json and clear caches.
* Prevents resource leaks and "Jest did not exit" warnings in tests.
*/
dispose() {
// storageLayer.clear() cancels pending _index.json debounced writes,
// preventing ENOTEMPTY errors when tests delete temp directories.
super.dispose(); // Calls BaseElementManager.dispose() — runs storageLayer.clear() + watcher cleanup
this.contentHashIndex.clear();
this.contentHashByPath.clear();
}
/**
* Resolve memory file path across different storage locations
* Searches in this order: system/, adapters/, date folders, root (legacy)
* Extracted method to reduce cognitive complexity (PR #7)
* @private
*/
async resolveMemoryPath(filePath) {
// Check if it's a relative path (no date folder)
if (!filePath.includes(path.sep) || !filePath.match(/^\d{4}-\d{2}-\d{2}/)) {
// Search in system/ folder first
const systemPath = path.join(this.memoriesDir, 'system', filePath);
if (await this.fileOperations.exists(systemPath)) {
return systemPath;
}
// Then check adapters/ folder
const adapterPath = path.join(this.memoriesDir, 'adapters', filePath);
if (await this.fileOperations.exists(adapterPath)) {
return adapterPath;
}
// Then search in date folders
const dateFolders = await this.getDateFolders();
for (const dateFolder of dateFolders) {
const testPath = path.join(this.memoriesDir, dateFolder, filePath);
if (await this.fileOperations.exists(testPath)) {
return testPath;
}
}
// Fall back to root directory for backward compatibility during transition
return await this.validateAndResolvePath(filePath);
}
else {
return await this.validateAndResolvePath(filePath);
}
}
/**
* Load a memory from file
* SECURITY FIX #1: Uses FileLockManager.atomicReadFile() instead of fs.readFile()
* to prevent race conditions and ensure atomic file operations
* @param filePath Path to the memory file to load
* @returns Promise resolving to the loaded Memory instance
* @throws {Error} When file cannot be found or path validation fails
* @throws {Error} When YAML parsing fails or content is malformed
* @throws {Error} When memory validation fails after loading
*/
async load(filePath) {
try {
// Resolve path using extracted helper method
const fullPath = await this.resolveMemoryPath(filePath);
// Ensure fullPath is defined
if (!fullPath) {
throw new Error(`Could not resolve path: ${filePath}`);
}
// Phase 2: Check base class LRU cache (replaces removed memoryCache Map)
const cached = this.getCachedByAbsolutePath(fullPath);
if (cached)
return cached;
// CRITICAL FIX: Use FileOperationsService for atomic file read
// Previously: const content = await fs.readFile(fullPath, 'utf-8');
// Now: Uses FileOperationsService which wraps FileLockManager
const content = await this.fileOperations.readFile(fullPath, { encoding: 'utf-8' });
// HIGH SEVERITY FIX: Use SecureYamlParser to prevent YAML injection attacks
// Uses SerializationService which wraps SecureYamlParser and handles pure YAML automatically
// Memory files are pure YAML (unlike other elements which are markdown with frontmatter)
// SerializationService.parseFrontmatter() automatically detects and wraps pure YAML
// Handle empty content edge case (for backward compatibility with existing tests/files)
let parsed;
if (content.trim() === '') {
// Empty or all whitespace file - create minimal valid structure
parsed = { data: {}, content: '' };
}
else {
const parseResult = this.serializationService.parseFrontmatter(content, {
maxYamlSize: MEMORY_CONSTANTS.MAX_YAML_SIZE,
validateContent: false, // FIX (#1206): Local files are pre-trusted
source: 'MemoryManager.load',
schema: 'json' // FIX #1430: Preserve booleans (autoLoad) and numbers (priority)
});
// Convert to ParsedMemoryData format
parsed = {
data: parseResult.data,
content: parseResult.content
};
}
// Extract metadata and markdown content
const { metadata, content: markdownContentFromFile } = this.parseMemoryFile(parsed);
// Create memory instance
const memory = new Memory(metadata, this.metadataService);
// Fix #918: Read instructions from root-level YAML (where serializeElement writes them).
// Previously instructions were written to root but never read back — silent data loss.
const rootInstructions = parsed.data?.instructions;
if (rootInstructions && typeof rootInstructions === 'string') {
memory.instructions = rootInstructions;
}
// Strip format_version from runtime metadata (Fix #912)
delete memory.metadata.format_version;
// Load saved entries if present
// Memory files have entries as a top-level key in the YAML
const entries = parsed.data?.entries;
if (entries) {
memory.deserialize(JSON.stringify({
id: memory.id,
type: memory.type,
version: memory.version,
metadata: memory.metadata,
extensions: memory.extensions,
entries: entries
}));
}
// If markdown content exists after the frontmatter, add it as a memory entry
// This preserves content from seed memories and memory files with markdown sections
if (markdownContentFromFile && markdownContentFromFile.trim() && parsed.content && parsed.content.trim()) {
await memory.addEntry(parsed.content.trim(), [], // tags
{ loadedAt: new Date().toISOString() }, // metadata
'file' // source
);
}
// FIX #1320: Set file path on memory for persistence (store relative path)
// Normalize to forward slashes so paths are consistent across platforms
// (path.relative() returns backslashes on Windows).
const relativePath = path.relative(this.memoriesDir, fullPath).split(path.sep).join('/');
memory.setFilePath(relativePath);
// Cache via base class LRU cache
this.cacheElement(memory, relativePath);
// Routine load — debug level only. Security event for MEMORY_LOADED was
// generating ~128K entries/session, overwhelming the 5K security ring buffer
// with 25x turnover (backpressure). Downgraded per Issue #1687 analysis.
logger.debug(`[MemoryManager] Loaded memory from ${path.basename(fullPath)}`);
return memory;
}
catch (error) {
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_LOAD_FAILED,
severity: 'MEDIUM',
source: 'MemoryManager.load',
details: `Failed to load memory from ${filePath}: ${error}`
});
throw new Error(`Failed to load memory: ${error}`);
}
}
/**
* Move existing memory file to user backup directory
* Issue #49: Prevents duplicate files by backing up instead of versioning
*/
async moveToUserBackup(existingPath) {
const date = new Date();
const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
const backupDir = path.join(this.memoriesDir, 'backups', 'user', dateFolder);
await this.fileOperations.createDirectory(backupDir);
const filename = path.basename(existingPath);
const timestamp = date.toISOString().replace(/[:.]/g, '-');
const backupName = filename.replace('.yaml', `.backup-${timestamp}.yaml`);
await this.fileOperations.renameFile(existingPath, path.join(backupDir, backupName));
logger.info(`[MemoryManager] Issue #49: Moved existing file to backup: ${backupName}`);
// Issue #654: Prune excess backups for this memory in this date folder
await this.pruneBackupsForMemory(backupDir, filename);
}
/**
* Prune excess backups for a specific memory in a date folder.
* Keeps only the N most recent backups (by timestamp in filename).
* Issue #654: Prevents unbounded backup growth (was 197k+ files).
*/
async pruneBackupsForMemory(backupDir, memoryFilename) {
const maxBackups = STORAGE_LAYER_CONFIG.MAX_BACKUPS_PER_MEMORY;
const baseName = memoryFilename.replace('.yaml', '');
try {
const allFiles = await this.fileOperations.listDirectory(backupDir);
// Find all backups for this specific memory
const memoryBackups = allFiles
.filter(f => f.startsWith(`${baseName}.backup-`) && f.endsWith('.yaml'))
.sort((a, b) => a.localeCompare(b)); // Lexicographic sort = chronological for ISO timestamps
if (memoryBackups.length <= maxBackups)
return;
// Delete oldest backups (keep the last N)
const toDelete = memoryBackups.slice(0, memoryBackups.length - maxBackups);
for (const file of toDelete) {
try {
await this.fileOperations.deleteFile(path.join(backupDir, file), ElementType.MEMORY, { source: 'MemoryManager.pruneBackupsForMemory' });
}
catch (error) {
logger.debug(`[MemoryManager] Failed to prune backup ${file}:`, error);
}
}
if (toDelete.length > 0) {
logger.info(`[MemoryManager] Issue #654: Pruned ${toDelete.length} excess backups for ${baseName} (kept ${maxBackups})`);
}
}
catch (error) {
// Non-fatal — backup pruning is best-effort
logger.debug(`[MemoryManager] Backup pruning failed for ${backupDir}:`, error);
}
}
/**
* Generate path for memory storage based on memory type
* Routes memories to appropriate folders:
* - SYSTEM: system/
* - ADAPTER: adapters/
* - USER: YYYY-MM-DD/ (date-based folders)
* @param element Memory element to save
* @param memoryType Type of memory (defaults to USER)
* @param fileName Optional custom filename
* @returns Full path to memory file
*/
async generateMemoryPath(element, memoryType, fileName) {
const type = memoryType || MemoryType.USER;
let targetDir;
// Route to appropriate folder based on memory type
switch (type) {
case MemoryType.SYSTEM:
targetDir = path.join(this.memoriesDir, 'system');
break;
case MemoryType.ADAPTER:
targetDir = path.join(this.memoriesDir, 'adapters');
break;
case MemoryType.USER:
default: {
// User memories go in date-based folders (existing behavior)
const date = new Date();
const dateFolder = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}`;
targetDir = path.join(this.memoriesDir, dateFolder);
break;
}
}
// Ensure target directory exists
await this.fileOperations.createDirectory(targetDir);
// Generate filename using unified normalization
// Note: Memory files use .yaml extension, so we use normalizeFilename() directly
const baseName = fileName || `${this.normalizeFilename(element.metadata.name || 'memory')}.yaml`;
// Issue #49: Check if file exists and move to backup if so (instead of version suffixes)
const targetPath = path.join(targetDir, baseName);
if (await this.fileOperations.exists(targetPath)) {
await this.moveToUserBackup(targetPath);
}
return targetPath;
}
/**
* Determine memory type from file path
* @param filePath Relative path within memories directory
* @returns Memory type based on path
*/
getMemoryTypeFromPath(filePath) {
if (filePath.startsWith('system/') || filePath.startsWith('system\\')) {
return MemoryType.SYSTEM;
}
if (filePath.startsWith('adapters/') || filePath.startsWith('adapters\\')) {
return MemoryType.ADAPTER;
}
return MemoryType.USER;
}
/**
* Ensure all required folder structure exists
* Creates system/, backups/system/, backups/user/, adapters/ folders
*/
async ensureFolderStructure() {
const folders = [
path.join(this.memoriesDir, 'system'),
path.join(this.memoriesDir, 'backups', 'system'),
path.join(this.memoriesDir, 'backups', 'user'),
path.join(this.memoriesDir, 'adapters')
];
for (const folder of folders) {
await this.fileOperations.createDirectory(folder);
}
}
/**
* Calculate SHA-256 hash of memory content for deduplication
* Implements Issue #994 - Content-based deduplication
*/
calculateContentHash(element) {
const content = JSON.stringify({
metadata: element.metadata,
entries: JSON.parse(element.serialize()).entries
});
return crypto.createHash('sha256').update(content).digest('hex');
}
/**
* Get all date folders in memories directory.
* Phase 2: Removed local cache — MemoryStorageLayer scan cooldown handles the hot path.
* This method is now only used by resolveMemoryPath() as a fallback.
* @returns Array of date folder names
*/
async getDateFolders() {
try {
const entries = await this.fileOperations.listDirectory(this.memoriesDir);
return entries
.filter(name => /^\d{4}-\d{2}-\d{2}$/.test(name))
.sort((a, b) => a.localeCompare(b))
.reverse(); // Most recent first
}
catch (error) {
if (error.code === 'ENOENT') {
return [];
}
throw error;
}
}
/**
* Save a memory to file
* SECURITY FIX #1: Uses FileLockManager.atomicWriteFile() for atomic operations
* @param element Memory element to save
* @param filePath Optional custom file path, defaults to type-based path
* @returns Promise that resolves when save is complete
* @throws {Error} When memory validation fails before saving
* @throws {Error} When path validation fails or file system errors occur
* @throws {Error} When atomic write operation fails
*/
async save(element, filePath) {
try {
// Issue #39: Auto-repair corrupted backup names before saving
const memoryName = element.metadata.name;
if (isCorruptedBackupName(memoryName)) {
const originalName = extractOriginalName(memoryName);
logger.warn(`[MemoryManager] Issue #39: Detected corrupted backup name '${memoryName}', ` +
`auto-repairing to '${originalName}'`);
element.metadata.name = originalName;
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_SAVED,
severity: 'MEDIUM',
source: 'MemoryManager.save',
details: `Issue #39: Auto-repaired corrupted backup name: '${memoryName}' -> '${originalName}'`
});
}
// Validate element
const validation = element.validate();
if (!validation.valid) {
throw new Error(`Invalid memory: ${validation.errors?.map(e => e.message).join(', ')}`);
}
// Calculate content hash for deduplication
const contentHash = this.calculateContentHash(element);
const existingPath = this.contentHashIndex.get(contentHash);
if (existingPath) {
// Log duplicate detection
SecurityMonitor.logSecurityEvent({
type: 'MEMORY_DUPLICATE_DETECTED',
severity: 'LOW',
source: 'MemoryManager.save',
details: `Duplicate content detected. Existing: ${existingPath}`
});
}
// Detect memory type from metadata or default to USER
const memoryMeta = element.metadata;
const memoryType = memoryMeta.memoryType || MemoryType.USER;
// Save path precedence:
// 1) explicit filePath argument
// 2) existing persisted path on the element (existingFilePath)
// 3) newly generated date/type-based path for first-time saves
// Keeping #2 ahead of #3 prevents loaded memories from being copied into a new
// date folder on each save (Issue #699).
const existingFilePath = element.getFilePath();
const fullPath = filePath
? await this.validateAndResolvePath(filePath)
: existingFilePath
? await this.validateAndResolvePath(existingFilePath)
: await this.generateMemoryPath(element, memoryType);
// Ensure parent directory exists
await this.fileOperations.createDirectory(path.dirname(fullPath));
const yamlContent = await this.serializeElement(element);
// Fix #916/#918: Size enforcement on Memory's custom save path
// (BaseElementManager.save() has this but Memory overrides it entirely)
if (yamlContent.length > SECURITY_LIMITS.MAX_PERSONA_SIZE_BYTES) {
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_SAVE_FAILED,
severity: 'HIGH',
source: 'MemoryManager.save.sizeEnforcement',
details: `Memory exceeds maximum file size (${yamlContent.length} > ${SECURITY_LIMITS.MAX_PERSONA_SIZE_BYTES})`,
metadata: { contentLength: yamlContent.length, limit: SECURITY_LIMITS.MAX_PERSONA_SIZE_BYTES }
});
throw new Error(`Memory exceeds maximum file size (${yamlContent.length} > ${SECURITY_LIMITS.MAX_PERSONA_SIZE_BYTES})`);
}
// Fix #908/#918: YAML bomb detection on Memory's custom save path
const validationStart = Date.now();
if (yamlContent.length <= SECURITY_LIMITS.MAX_YAML_LENGTH) {
if (!ContentValidator.validateYamlContent(yamlContent)) {
SecurityMonitor.logSecurityEvent({
type: 'YAML_INJECTION_ATTEMPT',
severity: 'CRITICAL',
source: 'MemoryManager.save.yamlBombDetection',
details: 'Serialized memory contains malicious YAML patterns — write blocked',
metadata: { contentLength: yamlContent.length }
});
throw new Error('Serialized memory contains malicious YAML patterns — write blocked');
}
}
const validationMs = Date.now() - validationStart;
if (validationMs > 50) {
logger.warn(`[MemoryManager] Write-path YAML validation took ${validationMs}ms for ${yamlContent.length} bytes`);
}
const parsedYaml = SecureYamlParser.parseRawYaml(yamlContent, SECURITY_LIMITS.MAX_YAML_LENGTH);
const gatekeeperErrors = [
...getGatekeeperAuthoringErrors(parsedYaml),
...getGatekeeperAuthoringErrors(parsedYaml.metadata && typeof parsedYaml.metadata === 'object' && !Array.isArray(parsedYaml.metadata)
? parsedYaml.metadata
: undefined),
];
if (gatekeeperErrors.length > 0) {
throw new Error(`Invalid gatekeeper policy in serialized memory YAML: ${[...new Set(gatekeeperErrors)].join('; ')}`);
}
// CRITICAL FIX: Use FileOperationsService for atomic file write
// Previously: await fs.writeFile(fullPath, yamlContent, 'utf-8');
// Now: Uses FileOperationsService which wraps FileLockManager
await this.fileOperations.writeFile(fullPath, yamlContent, { encoding: 'utf-8' });
// FIX #1320: Set file path on memory after successful save
// Normalize to forward slashes so paths are consistent across platforms
// (path.relative() returns backslashes on Windows).
const relativePath = path.relative(this.memoriesDir, fullPath).split(path.sep).join('/');
element.setFilePath(relativePath);
// Cache via base class LRU cache
this.cacheElement(element, relativePath);
// Phase 2: Notify storage layer of save
const relPath = path.relative(this.memoriesDir, fullPath).split(path.sep).join('/');
await this.storageLayer.notifySaved(relPath, fullPath);
// Update bounded content hash index
this.contentHashIndex.set(contentHash, fullPath);
this.contentHashByPath.set(fullPath, contentHash);
// Log successful save
const stats = element.getStats();
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_SAVED,
severity: 'LOW',
source: 'MemoryManager.save',
details: `Saved memory to ${path.basename(fullPath)} with ${stats.totalEntries} entries`
});
}
catch (error) {
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_SAVE_FAILED,
severity: 'HIGH',
source: 'MemoryManager.save',
details: `Failed to save memory to ${filePath}: ${error}`
});
throw new Error(`Failed to save memory: ${error}`);
}
}
/**
* Handle memory load failure
* FIX (SonarCloud): Extract duplicated error handling to reduce code duplication
* @private
*/
handleLoadFailure(file, error, failedLoads) {
const errorMsg = error instanceof Error ? error.message : String(error);
failedLoads.push({ file, error: errorMsg });
// Only log security event if load() didn't suppress this as a repeat
if (!this.isLoadErrorSuppressed(file)) {
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_LIST_ITEM_FAILED,
severity: 'LOW',
source: 'MemoryManager.list',
details: `Failed to load ${file}: ${error}`
});
}
}
/**
* List all available memories from all locations.
* Phase 2: Delegates multi-directory scanning to MemoryStorageLayer.
* Storage layer handles: system/, adapters/, date folders, root, backup filtering, cooldown.
*
* Issue #18 Phase 4: Apply active status to memories that are in the active set.
*/
async list() {
const failedLoads = [];
try {
await this.fileOperations.createDirectory(this.memoriesDir);
// Storage layer handles multi-directory scan + cooldown
const indexedPaths = await this.storageLayer.getIndexedPaths();
// Load through LRU cache
const memories = [];
for (const relativePath of indexedPaths) {
try {
const memory = await this.load(relativePath);
// Ensure memoryType is set based on location
const memoryMeta = memory.metadata;
if (!memoryMeta.memoryType) {
memoryMeta.memoryType = MemoryMetadataExtractor.inferMemoryType(relativePath);
}
memories.push(memory);
}
catch (error) {
this.handleLoadFailure(relativePath, error, failedLoads);
}
}
if (failedLoads.length > 0) {
logger.warn(`[MemoryManager] Failed to load ${failedLoads.length} memories:`, failedLoads.map(f => ` - ${f.file}: ${f.error}`).join('\n'));
}
// Deduplicate by file path (preserves existing behavior)
const uniqueMemories = new Map();
for (const memory of memories) {
const filePath = memory.getFilePath();
if (filePath && !uniqueMemories.has(filePath)) {
uniqueMemories.set(filePath, memory);
}
else if (!filePath) {
uniqueMemories.set(`no-path-${memory.metadata.name}-${Date.now()}`, memory);
}
}
const resultMemories = Array.from(uniqueMemories.values());
// Issue #18 Phase 4: Apply active status to memories in the active set
for (const memory of resultMemories) {
if (this.activeMemoryNames.has(memory.metadata.name)) {
await memory.activate();
}
}
return resultMemories;
}
catch (error) {
if (error.code === 'ENOENT') {
return [];
}
throw error;
}
}
/**
* Activate a memory by name or identifier
* Issue #18 Phase 4: Context-loading activation strategy
* Issue #24 (LOW PRIORITY): Performance optimization using findByName()
* Issue #24 (LOW PRIORITY): Consistent error messages using ElementMessages
* Issue #24 (LOW PRIORITY): Cleanup trigger for memory leak prevention
*/
async activateMemory(identifier) {
// PERFORMANCE FIX: Use findByName() instead of list()
const memory = await this.findByName(identifier);
if (!memory) {
return {
success: false,
// CONSISTENCY FIX: Use standardized error message format
message: ElementMessages.notFound(ElementType.MEMORY, identifier)
};
}
// MEMORY LEAK FIX: Check if cleanup is needed before adding
this.checkAndCleanupActiveSet();
// Add to active set (by name, which is stable across reloads)
this.activeMemoryNames.add(memory.metadata.name);
// Update memory status in memory
await memory.activate();
// Routine activation — debug only (was flooding security buffer)
logger.debug(`[MemoryManager] Memory activated: ${memory.metadata.name}`);
logger.info(`Memory activated: ${memory.metadata.name}`);
return {
success: true,
// CONSISTENCY FIX: Use standardized success message format
message: ElementMessages.activated(ElementType.MEMORY, memory.metadata.name),
memory
};
}
/**
* Deactivate a memory by name or identifier
* Issue #18 Phase 4: Remove from active set
* Issue #24 (LOW PRIORITY): Performance optimization using findByName()
* Issue #24 (LOW PRIORITY): Consistent error messages using ElementMessages
*/
async deactivateMemory(identifier) {
// PERFORMANCE FIX: Use findByName() instead of list()
const memory = await this.findByName(identifier);
if (!memory) {
return {
success: false,
// CONSISTENCY FIX: Use standardized error message format
message: ElementMessages.notFound(ElementType.MEMORY, identifier)
};
}
// Remove from active set
this.activeMemoryNames.delete(memory.metadata.name);
// Update memory status in memory
await memory.deactivate();
logger.info(`Memory deactivated: ${memory.metadata.name}`);
return {
success: true,
// CONSISTENCY FIX: Use standardized success message format
message: ElementMessages.deactivated(ElementType.MEMORY, memory.metadata.name)
};
}
/**
* Get all active memories
* Issue #18 Phase 4: Return memories that are in the active set
*/
async getActiveMemories() {
const memories = await this.list();
return memories.filter(m => this.activeMemoryNames.has(m.metadata.name));
}
/**
* Issue #39: Scan all memories and repair corrupted backup names
* This method loads all memories, checks for corrupted names, repairs them,
* and saves the corrected files back to disk.
*
* @param onProgress Optional callback for progress updates during long operations
* @returns Object with repair statistics and list of repaired memories
*/
async repairCorruptedNames(onProgress) {
const result = {
scanned: 0,
repaired: 0,
errors: 0,
repairedMemories: [],
errorDetails: []
};
logger.info('[MemoryManager] Issue #39: Starting corrupted name repair scan...');
try {
// Force fresh scan — repair must see current disk state, not cached index
this.storageLayer.invalidate();
const memories = await this.list();
result.scanned = memories.length;
let processed = 0;
for (const memory of memories) {
const currentName = memory.metadata.name;
processed++;
// Report progress
if (onProgress) {
onProgress(processed, memories.length, currentName);
}
if (isCorruptedBackupName(currentName)) {
const repairedName = extractOriginalName(currentName);
const filePath = memory.getFilePath() || 'unknown';
try {
// Update the name
memory.metadata.name = repairedName;
// Save the repaired memory
await this.save(memory);
result.repaired++;
result.repairedMemories.push({
original: currentName,
repaired: repairedName,
path: filePath
});
logger.info(`[MemoryManager] Issue #39: Repaired '${currentName}' -> '${repairedName}' at ${filePath}`);
}
catch (error) {
result.errors++;
result.errorDetails.push({
name: currentName,
error: error instanceof Error ? error.message : String(error)
});
logger.error(`[MemoryManager] Issue #39: Failed to repair '${currentName}':`, error);
}
}
}
logger.info(`[MemoryManager] Issue #39: Repair scan complete. ` +
`Scanned: ${result.scanned}, Repaired: ${result.repaired}, Errors: ${result.errors}`);
if (result.repaired > 0) {
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_SAVED,
severity: 'MEDIUM',
source: 'MemoryManager.repairCorruptedNames',
details: `Issue #39: Repaired ${result.repaired} corrupted memory names`,
additionalData: {
scanned: result.scanned,
repaired: result.repaired,
errors: result.errors
}
});
}
return result;
}
catch (error) {
logger.error('[MemoryManager] Issue #39: Repair scan failed:', error);
throw error;
}
}
/**
* Issue #39: Clean up excessive backup files in a directory
* Removes versioned backup files (e.g., name.backup-...-v2.yaml, -v3.yaml, etc.)
* keeping only the most recent backup (without version suffix)
*
* @param targetDir Directory to clean up (defaults to system/ folder)
* @param dryRun If true, only reports what would be deleted without actually deleting
* @param onProgress Optional callback for progress updates during long operations
* @returns Object with cleanup statistics
*/
async cleanupExcessiveBackups(targetDir, dryRun = false, onProgress) {
const dir = targetDir || path.join(this.memoriesDir, 'system');
const result = {
scanned: 0,
deleted: 0,
errors: 0,
deletedFiles: [],
keptFiles: [],
errorDetails: []
};
logger.info(`[MemoryManager] Issue #39: Starting backup cleanup in ${dir}${dryRun ? ' (DRY RUN)' : ''}...`);
try {
const files = await this.fileOperations.listDirectory(dir);
result.scanned = files.length;
// Group backup files by base name
const backupGroups = new Map();
for (const file of files) {
// Match versioned backup files: name.backup-YYYY-MM-DD-HH-mm-ss-SSS-vN.yaml
const versionedMatch = file.match(/^(.+\.backup-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-\d{3})-v(\d+)\.yaml$/);
if (versionedMatch) {
const baseName = versionedMatch[1];
const existing = backupGroups.get(baseName) || [];
existing.push(file);
backupGroups.set(baseName, existing);
}
}
// Delete all versioned backup files
let processed = 0;
const totalVersionedFiles = Array.from(backupGroups.values()).reduce((sum, files) => sum + files.length, 0);
for (const [baseName, versionedFiles] of backupGroups) {
// Keep the base backup file (without -vN), delete all versioned ones
const baseBackupFile = `${baseName}.yaml`;
if (files.includes(baseBackupFile)) {
result.keptFiles.push(baseBackupFile);
}
for (const versionedFile of versionedFiles) {
processed++;
// Report progress
if (onProgress) {
onProgress(processed, totalVersionedFiles, versionedFile);
}
const fullPath = path.join(dir, versionedFile);
try {
if (!dryRun) {
await this.fileOperations.deleteFile(fullPath, ElementType.MEMORY, {
source: 'MemoryManager.cleanupExcessiveBackups'
});
}
result.deleted++;
result.deletedFiles.push(versionedFile);
logger.debug(`[MemoryManager] Issue #39: ${dryRun ? 'Would delete' : 'Deleted'} ${versionedFile}`);
}
catch (error) {
result.errors++;
result.errorDetails.push({
file: versionedFile,
error: error instanceof Error ? error.message : String(error)
});
logger.error(`[MemoryManager] Issue #39: Failed to delete ${versionedFile}:`, error);
}
}
}
logger.info(`[MemoryManager] Issue #39: Backup cleanup complete${dryRun ? ' (DRY RUN)' : ''}. ` +
`Scanned: ${result.scanned}, ${dryRun ? 'Would delete' : 'Deleted'}: ${result.deleted}, Errors: ${result.errors}`);
if (result.deleted > 0 && !dryRun) {
SecurityMonitor.logSecurityEvent({
type: MEMORY_SECURITY_EVENTS.MEMORY_CLEARED,
severity: 'LOW',
source: 'MemoryManager.cleanupExcessiveBackups',
details: `Issue #39: Cleaned up ${result.deleted} excessive backup files`,
additionalData: {
directory: dir,
scanned: result.scanned,
deleted: result.deleted,
errors: result.errors
}
});
}
return result;
}
catch (error) {
logger.error('[MemoryManager] Issue #39: Backup cleanup failed:', error);
throw error;
}
}
/**
* Check if active set cleanup is needed and perform cleanup if necessary
* Issue #24 (LOW PRIORITY): Memory leak prevention
* @private
*/
checkAndCleanupActiveSet() {
const { max, cleanupThreshold } = getActiveElementLimitConfig('memories');
// Below threshold — no action needed
if (this.activeMemoryNames.size < cleanupThreshold) {
return;
}
// At or above max — warn before cleanup
if (this.activeMemoryNames.size >= max) {
logger.warn(`Active memories limit reached (${max}). ` +
`Consider deactivating unused memories or setting DOLLHOUSE_MAX_ACTIVE_MEMORIES to a higher value.`);
SecurityMonitor.logSecurityEvent({
type: 'MEMORY_LOADED',
severity: 'MEDIUM',
source: 'MemoryManager.checkAndCleanupActiveSet',
details: `Active memories limit reached: ${this.activeMemoryNames.size}/${max}`
});
}
// At or above threshold — proactively clean stale entries
void this.cleanupStaleActiveMemories();
}
/**
* Clean up stale entries from active memories set
* Issue #24 (LOW PRIORITY): Memory leak prevention
* @private
*/
async cleanupStaleActiveMemories() {
try {
const startSize = this.activeMemoryNames.size;
const memories = await this.list();
const existingMemoryNames = new Set(memories.map(m => m.metadata.name));
const staleNames = [];
for (const activeName of this.activeMemoryNames) {
if (!existingMemoryNames.has(activeName)) {
this.activeMemoryNames.delete(activeName);
staleNames.push(activeName);
}
}
const endSize = this.activeMemoryNames.size;
const removed = startSize - endSize;
if (removed > 0) {
logger.info(`Cleaned up ${removed} stale active memory reference(s). ` +
`Active memories: ${endSize}/${getMaxActiveLimit('memories')}`);
SecurityMonitor.logSecurityEvent({
type: 'MEMORY_DELETED',
severity: 'LOW',
source: 'MemoryManager.cleanupStaleActiveMemories',
details: `Removed ${removed} stale active memory references`,
additionalData: {
removedCount: removed,
activeCount: endSize,
staleNames: staleNames.join(', ')
}
});
}
}
catch (error) {
logger.error('Failed to cleanup stale active memories:', error);
SecurityMonitor.logSecurityEvent({
type: 'MEMORY_DELETED',
severity: 'LOW',
source: 'MemoryManager.cleanupStaleActiveMemories',
details: `Cleanup failed: ${error instanceof Error ? error.message : String(error)}`
});
}
}
/**
* Check root files for load failures
* FIX (SonarCloud S3776): Extract to reduce cognitive complexity
* @private
*/
async checkRootFiles(failures) {
const rootFiles = await this.fileOperations.listDirectory(this.memoriesDir)
.then(files => files.filter(f => f.endsWith('.yaml')))
.catch(() => []);
for (const file of rootFiles) {
try {
await this.load(file);
}
catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
failures.push({ file, error: errorMsg });
}
}
return rootFiles.length;
}
/**
* Check date folder files for load failures
* FIX (SonarCloud S3776): Extract to reduce cognitive complexity
* @private
*/
async checkDateFolderFiles(dateFolder, failures) {
const files = await this.fileOperations.listDirectory(path.join(this.memoriesDir, dateFolder))
.then(files => files.filter(f => f.endsWith('.yaml')))
.catch(() => []);
for (const file of files) {
try {
await this.load(path.join(dateFolder, file));
}
catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
failures.push({ file: `${dateFolder}/${file}`, error: errorMsg });
}
}
return files.length;
}
/**
* Get diagnostic information about memory loading status
* FIX (#1206): New method to expose failed loads to users
*/
async getLoadStatus() {
const failures = [];
let totalFiles = 0;
try {
const dateFolders = await this.getDateFolders();
// Check root files
totalFiles += await this.checkRootFiles(failures);
// Check date folders
for (const dateFolder of dateFolders) {
totalFiles += await this.checkDateFolderFiles(dateFolder, failures);
}
return {
total: totalFiles,
loaded: totalFiles - failures.length,
failed: failures.length,
failures
};
}
catch (error) {
throw new Error(`Failed to get load status: ${error}`);
}
}
/**
* Get memories marked for auto-loading on server initialization.
* Phase 2: Queries index for autoLoad entries instead of loading all memories.
* Drops from ~70s to <30ms for typical portfolios with 14k memories.
* Issue #1430: Auto-load baseline memories feature
*
* @returns Promise resolving to array of auto-load memories sorted by priority
*/
async getAutoLoadMemories() {
try {
// Ensure index is populated