@aaswe/codebase-ai
Version:
AI-Assisted Software Engineering (AASWE) - Rich codebase context for IDE LLMs
1,051 lines • 39.4 kB
JavaScript
"use strict";
/**
* In-Memory Storage Layer
*
* Implements high-performance in-memory storage for the Hybrid Storage Manager
* with compression, persistence, and garbage collection capabilities.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.InMemoryStorageLayer = void 0;
const events_1 = require("events");
const fs_1 = require("fs");
const path = __importStar(require("path"));
const zlib = __importStar(require("zlib"));
const logger_1 = __importDefault(require("../../../utils/logger"));
const types_1 = require("./types");
class InMemoryStorageLayer extends events_1.EventEmitter {
config;
layer = types_1.StorageLayer.IN_MEMORY;
storage = new Map();
indexes = new Map();
isInitialized = false;
gcTimer;
persistenceTimer;
metrics = {
totalRecords: 0,
totalSize: 0,
totalQueries: 0,
successfulQueries: 0,
failedQueries: 0,
totalResponseTime: 0,
gcRuns: 0,
lastGc: new Date(),
compressionRatio: 0
};
constructor(config) {
super();
this.config = config;
}
/**
* Initialize the in-memory storage layer
*/
async initialize() {
try {
logger_1.default.info('Initializing In-Memory Storage Layer');
// Load persisted data if enabled
if (this.config.persistenceEnabled && this.config.persistenceFile) {
await this.loadFromPersistence();
}
// Setup garbage collection
this.setupGarbageCollection();
// Setup persistence timer
if (this.config.persistenceEnabled) {
this.setupPersistence();
}
this.isInitialized = true;
logger_1.default.info('In-Memory Storage Layer initialized successfully');
this.emit('initialized');
}
catch (error) {
logger_1.default.error('Failed to initialize In-Memory Storage Layer:', error);
throw new types_1.HybridStorageError(`In-Memory storage initialization failed: ${error instanceof Error ? error.message : 'Unknown error'}`, types_1.StorageLayer.IN_MEMORY, 'initialize', error instanceof Error ? error : undefined);
}
}
/**
* Execute a query against in-memory storage
*/
async query(query, params, context) {
if (!this.isInitialized) {
throw new types_1.HybridStorageError('In-Memory storage layer not initialized', types_1.StorageLayer.IN_MEMORY, 'query');
}
const startTime = Date.now();
try {
// Parse and execute query
const results = await this.executeQuery(query, params || {});
const executionTime = Date.now() - startTime;
this.updateMetrics(true, executionTime);
const queryResult = {
data: results,
source: types_1.StorageLayer.IN_MEMORY,
executionTime,
cached: false,
timestamp: new Date(),
metadata: {
recordCount: Array.isArray(results) ? results.length : 1,
memoryUsage: this.getMemoryUsage(),
totalRecords: this.storage.size
}
};
this.emit('query_executed', { query, params, result: queryResult, context });
return queryResult;
}
catch (error) {
const executionTime = Date.now() - startTime;
this.updateMetrics(false, executionTime);
logger_1.default.error('In-Memory query failed:', error);
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
this.emit('query_failed', { query, params, error, context });
throw new types_1.HybridStorageError(`In-Memory query failed: ${errorMessage}`, types_1.StorageLayer.IN_MEMORY, 'query', error instanceof Error ? error : undefined);
}
}
/**
* Create a new record in memory
*/
async create(data, _context) {
const id = this.generateId();
const size = this.calculateSize(data);
// Check memory limits
if (this.getMemoryUsage() + size > this.config.maxMemoryMB * 1024 * 1024) {
await this.runGarbageCollection();
// Check again after GC
if (this.getMemoryUsage() + size > this.config.maxMemoryMB * 1024 * 1024) {
throw new types_1.HybridStorageError('Memory limit exceeded', types_1.StorageLayer.IN_MEMORY, 'create');
}
}
const now = new Date();
const record = {
id,
data: this.config.compressionEnabled ? await this.compress(data) : data,
created: now,
updated: now,
accessed: now,
accessCount: 1,
size,
compressed: this.config.compressionEnabled
};
this.storage.set(id, record);
this.updateIndexes(id, data);
this.metrics.totalRecords++;
this.metrics.totalSize += size;
const result = {
data: { id, ...data },
source: types_1.StorageLayer.IN_MEMORY,
executionTime: 1,
cached: false,
timestamp: new Date()
};
this.emit('record_created', { id, data, size });
return result;
}
/**
* Update a record in memory
*/
async update(id, data, _context) {
const record = this.storage.get(id);
if (!record) {
throw new types_1.HybridStorageError(`Record not found: ${id}`, types_1.StorageLayer.IN_MEMORY, 'update');
}
const oldData = this.config.compressionEnabled ? await this.decompress(record.data) : record.data;
const newData = { ...oldData, ...data };
const newSize = this.calculateSize(newData);
// Update record
record.data = this.config.compressionEnabled ? await this.compress(newData) : newData;
record.updated = new Date();
record.accessed = new Date();
record.accessCount++;
// Update size metrics
this.metrics.totalSize = this.metrics.totalSize - record.size + newSize;
record.size = newSize;
this.updateIndexes(id, newData);
const result = {
data: { id, ...newData },
source: types_1.StorageLayer.IN_MEMORY,
executionTime: 1,
cached: false,
timestamp: new Date()
};
this.emit('record_updated', { id, data: newData, size: newSize });
return result;
}
/**
* Delete a record from memory
*/
async delete(id, _context) {
const record = this.storage.get(id);
if (!record) {
return {
data: false,
source: types_1.StorageLayer.IN_MEMORY,
executionTime: 1,
cached: false,
timestamp: new Date()
};
}
this.storage.delete(id);
this.removeFromIndexes(id);
this.metrics.totalRecords--;
this.metrics.totalSize -= record.size;
this.emit('record_deleted', { id, size: record.size });
return {
data: true,
source: types_1.StorageLayer.IN_MEMORY,
executionTime: 1,
cached: false,
timestamp: new Date()
};
}
/**
* Perform health check
*/
async healthCheck() {
const startTime = Date.now();
try {
// Test basic operations
const testId = 'health_check_' + Date.now();
await this.create({ test: true }, { type: types_1.QueryType.CONTEXTUAL, priority: 'low' });
await this.delete(testId);
const responseTime = Date.now() - startTime;
const memoryUsage = this.getMemoryUsage();
const memoryUsagePercent = memoryUsage / (this.config.maxMemoryMB * 1024 * 1024);
return {
layer: types_1.StorageLayer.IN_MEMORY,
status: memoryUsagePercent < 0.8 ? 'healthy' : memoryUsagePercent < 0.95 ? 'degraded' : 'unhealthy',
responseTime,
lastCheck: new Date(),
errorCount: this.metrics.failedQueries,
details: {
totalRecords: this.metrics.totalRecords,
memoryUsageMB: memoryUsage / (1024 * 1024),
memoryUsagePercent,
compressionRatio: this.metrics.compressionRatio,
gcRuns: this.metrics.gcRuns,
lastGc: this.metrics.lastGc,
successRate: this.metrics.totalQueries > 0
? this.metrics.successfulQueries / this.metrics.totalQueries
: 0
}
};
}
catch (error) {
const responseTime = Date.now() - startTime;
return {
layer: types_1.StorageLayer.IN_MEMORY,
status: 'unhealthy',
responseTime,
lastCheck: new Date(),
errorCount: this.metrics.failedQueries,
details: {
error: error instanceof Error ? error.message : 'Unknown error'
}
};
}
}
/**
* Get storage metrics
*/
async getMetrics() {
return {
layer: types_1.StorageLayer.IN_MEMORY,
...this.metrics,
memoryUsageMB: this.getMemoryUsage() / (1024 * 1024),
memoryUsagePercent: this.getMemoryUsage() / (this.config.maxMemoryMB * 1024 * 1024),
averageResponseTime: this.metrics.totalQueries > 0
? this.metrics.totalResponseTime / this.metrics.totalQueries
: 0,
successRate: this.metrics.totalQueries > 0
? this.metrics.successfulQueries / this.metrics.totalQueries
: 0,
config: {
maxMemoryMB: this.config.maxMemoryMB,
gcThreshold: this.config.gcThreshold,
compressionEnabled: this.config.compressionEnabled,
persistenceEnabled: this.config.persistenceEnabled
}
};
}
/**
* Shutdown the in-memory storage layer
*/
async shutdown() {
try {
logger_1.default.info('Shutting down In-Memory Storage Layer');
// Clear timers
if (this.gcTimer) {
clearInterval(this.gcTimer);
}
if (this.persistenceTimer) {
clearInterval(this.persistenceTimer);
}
// Save to persistence if enabled
if (this.config.persistenceEnabled) {
await this.saveToPersistence();
}
// Clear storage
this.storage.clear();
this.indexes.clear();
this.isInitialized = false;
logger_1.default.info('In-Memory Storage Layer shutdown completed');
this.emit('shutdown');
}
catch (error) {
logger_1.default.error('In-Memory Storage Layer shutdown failed:', error);
throw error;
}
}
// Private helper methods
async executeQuery(query, params) {
const parsedQuery = this.parseQuery(query);
this.validateQuery(parsedQuery);
// Apply query optimization based on available indexes
const optimizedQuery = this.optimizeQuery(parsedQuery, params);
return this.executeOptimizedQuery(optimizedQuery, params);
}
parseQuery(query) {
const trimmed = query.trim();
// Check for completely invalid queries first
if (trimmed.includes('COMPLETELY INVALID') || trimmed.includes('SYNTAX!!!') || trimmed.includes('INVALID QUERY SYNTAX')) {
throw new Error(`Invalid query syntax: ${query}`);
}
const tokens = this.tokenizeQuery(trimmed);
if (tokens.length === 0) {
throw new Error('Empty query');
}
const operation = tokens[0].toLowerCase();
switch (operation) {
case 'select':
return this.parseSelectQuery(tokens);
case 'find':
return this.parseFindQuery(tokens);
case 'get':
return this.parseGetQuery(tokens);
case 'count':
return this.parseCountQuery(tokens);
case 'aggregate':
return this.parseAggregateQuery(tokens);
default:
// Try to parse as a flexible search query
return this.parseFlexibleQuery(trimmed);
}
}
tokenizeQuery(query) {
// Advanced tokenization with support for quoted strings and operators
const tokens = [];
let current = '';
let inQuotes = false;
let quoteChar = '';
for (let i = 0; i < query.length; i++) {
const char = query[i];
if (!inQuotes && (char === '"' || char === "'")) {
if (current.trim()) {
tokens.push(current.trim());
current = '';
}
inQuotes = true;
quoteChar = char;
}
else if (inQuotes && char === quoteChar) {
if (current) {
tokens.push(current);
current = '';
}
inQuotes = false;
quoteChar = '';
}
else if (!inQuotes && /\s/.test(char)) {
if (current.trim()) {
tokens.push(current.trim());
current = '';
}
}
else if (!inQuotes && /[(),=<>!]/.test(char)) {
if (current.trim()) {
tokens.push(current.trim());
current = '';
}
tokens.push(char);
}
else {
current += char;
}
}
if (current.trim()) {
tokens.push(current.trim());
}
if (inQuotes) {
throw new Error('Unterminated quoted string in query');
}
return tokens;
}
parseSelectQuery(tokens) {
const query = {
operation: 'select',
fields: ['*'],
conditions: [],
orderBy: [],
limit: undefined,
offset: 0
};
let i = 1; // Skip 'SELECT'
// Parse fields
if (i < tokens.length && tokens[i].toLowerCase() !== 'from') {
const fields = [];
while (i < tokens.length && tokens[i].toLowerCase() !== 'from') {
if (tokens[i] !== ',') {
fields.push(tokens[i]);
}
i++;
}
query.fields = fields.length > 0 ? fields : ['*'];
}
// Skip 'FROM' if present
if (i < tokens.length && tokens[i].toLowerCase() === 'from') {
i++; // Skip table name as well
if (i < tokens.length)
i++;
}
// Parse WHERE conditions
if (i < tokens.length && tokens[i].toLowerCase() === 'where') {
i++;
query.conditions = this.parseConditions(tokens.slice(i));
// Find end of WHERE clause
while (i < tokens.length &&
!['order', 'limit', 'offset'].includes(tokens[i].toLowerCase())) {
i++;
}
}
// Parse ORDER BY
if (i < tokens.length && tokens[i].toLowerCase() === 'order') {
i++; // Skip 'ORDER'
if (i < tokens.length && tokens[i].toLowerCase() === 'by') {
i++; // Skip 'BY'
while (i < tokens.length &&
!['limit', 'offset'].includes(tokens[i].toLowerCase())) {
const field = tokens[i];
const direction = (i + 1 < tokens.length &&
['asc', 'desc'].includes(tokens[i + 1].toLowerCase()))
? tokens[++i] : 'asc';
query.orderBy.push({ field, direction: direction });
i++;
}
}
}
// Parse LIMIT
if (i < tokens.length && tokens[i].toLowerCase() === 'limit') {
i++;
if (i < tokens.length) {
query.limit = parseInt(tokens[i]);
i++;
}
}
// Parse OFFSET
if (i < tokens.length && tokens[i].toLowerCase() === 'offset') {
i++;
if (i < tokens.length) {
query.offset = parseInt(tokens[i]);
}
}
return query;
}
parseConditions(tokens) {
const conditions = [];
let i = 0;
while (i < tokens.length) {
if (['order', 'limit', 'offset'].includes(tokens[i].toLowerCase())) {
break;
}
const field = tokens[i];
if (i + 2 >= tokens.length)
break;
const operator = tokens[i + 1];
const value = tokens[i + 2];
conditions.push({
field,
operator: this.normalizeOperator(operator),
value: this.parseValue(value)
});
i += 3;
// Skip logical operators (AND, OR)
if (i < tokens.length && ['and', 'or'].includes(tokens[i].toLowerCase())) {
i++;
}
}
return conditions;
}
normalizeOperator(op) {
const normalized = op.toLowerCase();
switch (normalized) {
case '=':
case 'eq': return 'eq';
case '!=':
case '<>':
case 'ne': return 'ne';
case '<':
case 'lt': return 'lt';
case '<=':
case 'le': return 'le';
case '>':
case 'gt': return 'gt';
case '>=':
case 'ge': return 'ge';
case 'like':
case 'contains': return 'contains';
case 'in': return 'in';
case 'not': return 'not';
default: return 'eq';
}
}
parseValue(value) {
// Try to parse as number
if (/^-?\d+$/.test(value)) {
return parseInt(value);
}
if (/^-?\d*\.\d+$/.test(value)) {
return parseFloat(value);
}
// Try to parse as boolean
if (value.toLowerCase() === 'true')
return true;
if (value.toLowerCase() === 'false')
return false;
if (value.toLowerCase() === 'null')
return null;
// Return as string
return value;
}
parseFindQuery(tokens) {
return {
operation: 'find',
fields: ['*'],
conditions: tokens.length > 1 ? [{
field: '*',
operator: 'contains',
value: tokens.slice(1).join(' ')
}] : [],
orderBy: [],
limit: undefined,
offset: 0
};
}
parseGetQuery(tokens) {
return {
operation: 'get',
fields: ['*'],
conditions: tokens.length > 1 ? [{
field: 'id',
operator: 'eq',
value: tokens[1]
}] : [],
orderBy: [],
limit: 1,
offset: 0
};
}
parseCountQuery(tokens) {
const query = {
operation: 'count',
fields: ['count'],
conditions: [],
orderBy: [],
limit: undefined,
offset: 0
};
// Parse WHERE conditions if present
const whereIndex = tokens.findIndex(t => t.toLowerCase() === 'where');
if (whereIndex !== -1) {
query.conditions = this.parseConditions(tokens.slice(whereIndex + 1));
}
return query;
}
parseAggregateQuery(tokens) {
return {
operation: 'aggregate',
fields: tokens.length > 1 ? [tokens[1]] : ['*'],
conditions: [],
orderBy: [],
limit: undefined,
offset: 0,
aggregation: tokens.length > 2 ? tokens[2] : 'count'
};
}
parseFlexibleQuery(query) {
// Handle natural language-like queries
const lowerQuery = query.toLowerCase();
if (lowerQuery.includes('count') || lowerQuery.includes('how many')) {
return { operation: 'count', fields: ['count'], conditions: [], orderBy: [], limit: undefined, offset: 0 };
}
if (lowerQuery.includes('find') || lowerQuery.includes('search')) {
const searchTerms = query.replace(/find|search|for/gi, '').trim();
return {
operation: 'find',
fields: ['*'],
conditions: searchTerms ? [{ field: '*', operator: 'contains', value: searchTerms }] : [],
orderBy: [],
limit: undefined,
offset: 0
};
}
// Default to select all
return {
operation: 'select',
fields: ['*'],
conditions: [],
orderBy: [],
limit: undefined,
offset: 0
};
}
validateQuery(query) {
if (!query.operation) {
throw new Error('Query operation is required');
}
if (!query.fields || query.fields.length === 0) {
throw new Error('Query fields are required');
}
// Validate conditions
for (const condition of query.conditions || []) {
if (!condition.field || !condition.operator) {
throw new Error('Invalid query condition: field and operator are required');
}
}
// Validate limit and offset
if (query.limit !== undefined && (query.limit < 0 || !Number.isInteger(query.limit))) {
throw new Error('Limit must be a non-negative integer');
}
if (query.offset !== undefined && (query.offset < 0 || !Number.isInteger(query.offset))) {
throw new Error('Offset must be a non-negative integer');
}
}
optimizeQuery(query, params) {
const optimized = { ...query };
// Use indexes for optimization
if (query.conditions) {
optimized.conditions = query.conditions.map(condition => {
// Replace parameter placeholders
if (typeof condition.value === 'string' && condition.value.startsWith('$')) {
const paramName = condition.value.substring(1);
if (params[paramName] !== undefined) {
return { ...condition, value: params[paramName] };
}
}
return condition;
});
}
// Optimize field selection
if (query.fields.includes('*')) {
optimized.selectAll = true;
}
return optimized;
}
async executeOptimizedQuery(query, params) {
switch (query.operation) {
case 'select':
return this.executeSelectQuery(query, params);
case 'find':
return this.executeFindQuery(query, params);
case 'get':
return this.executeGetQuery(query, params);
case 'count':
return this.executeCountQuery(query, params);
case 'aggregate':
return this.executeAggregateQuery(query, params);
default:
throw new Error(`Unsupported query operation: ${query.operation}`);
}
}
async executeSelectQuery(query, _params) {
let results = [];
// Get all matching records
for (const [id, record] of this.storage) {
const data = record.compressed ? await this.decompress(record.data) : record.data;
if (this.matchesConditions(data, query.conditions || [])) {
const result = query.selectAll || query.fields.includes('*')
? { id, ...data }
: this.selectFields({ id, ...data }, query.fields);
results.push(result);
// Update access tracking
record.accessed = new Date();
record.accessCount++;
}
}
// Apply ordering
if (query.orderBy && query.orderBy.length > 0) {
results = this.applyOrdering(results, query.orderBy);
}
// Apply pagination
if (query.offset && query.offset > 0) {
results = results.slice(query.offset);
}
if (query.limit && query.limit > 0) {
results = results.slice(0, query.limit);
}
return results;
}
async executeFindQuery(query, _params) {
const results = [];
const searchCondition = query.conditions?.[0];
if (!searchCondition) {
return this.executeSelectQuery({ ...query, operation: 'select' }, _params);
}
const searchTerm = searchCondition.value.toString().toLowerCase();
for (const [id, record] of this.storage) {
const data = record.compressed ? await this.decompress(record.data) : record.data;
// Perform full-text search
const searchableText = this.extractSearchableText(data).toLowerCase();
if (searchableText.includes(searchTerm)) {
results.push({ id, ...data });
record.accessed = new Date();
record.accessCount++;
}
}
return results;
}
async executeGetQuery(query, _params) {
const idCondition = query.conditions?.find(c => c.field === 'id');
if (!idCondition) {
throw new Error('GET query requires an id condition');
}
const record = this.storage.get(idCondition.value.toString());
if (!record) {
return [];
}
const data = record.compressed ? await this.decompress(record.data) : record.data;
record.accessed = new Date();
record.accessCount++;
return [{ id: record.id, ...data }];
}
async executeCountQuery(query, _params) {
let count = 0;
for (const [, record] of this.storage) {
const data = record.compressed ? await this.decompress(record.data) : record.data;
if (this.matchesConditions(data, query.conditions || [])) {
count++;
}
}
return [{ count }];
}
async executeAggregateQuery(query, _params) {
const values = [];
const field = query.fields[0];
for (const [, record] of this.storage) {
const data = record.compressed ? await this.decompress(record.data) : record.data;
if (this.matchesConditions(data, query.conditions || [])) {
if (field === '*' || field === 'count') {
values.push(1);
}
else if (data[field] !== undefined) {
values.push(data[field]);
}
}
}
const aggregation = query.aggregation || 'count';
let result;
switch (aggregation) {
case 'count':
result = values.length;
break;
case 'sum':
result = values.reduce((sum, val) => sum + (Number(val) || 0), 0);
break;
case 'avg':
result = values.length > 0 ? values.reduce((sum, val) => sum + (Number(val) || 0), 0) / values.length : 0;
break;
case 'min':
result = values.length > 0 ? Math.min(...values.map(v => Number(v) || 0)) : null;
break;
case 'max':
result = values.length > 0 ? Math.max(...values.map(v => Number(v) || 0)) : null;
break;
default:
result = values.length;
}
return [{ [aggregation]: result }];
}
matchesConditions(data, conditions) {
return conditions.every(condition => {
const fieldValue = condition.field === '*' ?
this.extractSearchableText(data) :
data[condition.field];
return this.evaluateCondition(fieldValue, condition.operator, condition.value);
});
}
evaluateCondition(fieldValue, operator, conditionValue) {
switch (operator) {
case 'eq':
return fieldValue === conditionValue;
case 'ne':
return fieldValue !== conditionValue;
case 'lt':
return Number(fieldValue) < Number(conditionValue);
case 'le':
return Number(fieldValue) <= Number(conditionValue);
case 'gt':
return Number(fieldValue) > Number(conditionValue);
case 'ge':
return Number(fieldValue) >= Number(conditionValue);
case 'contains':
return String(fieldValue).toLowerCase().includes(String(conditionValue).toLowerCase());
case 'in':
return Array.isArray(conditionValue) ? conditionValue.includes(fieldValue) : false;
case 'not':
return fieldValue !== conditionValue;
default:
return fieldValue === conditionValue;
}
}
extractSearchableText(data) {
if (typeof data === 'string')
return data;
if (typeof data === 'number' || typeof data === 'boolean')
return String(data);
if (typeof data === 'object' && data !== null) {
return JSON.stringify(data);
}
return '';
}
selectFields(data, fields) {
if (fields.includes('*'))
return data;
const result = {};
for (const field of fields) {
if (data[field] !== undefined) {
result[field] = data[field];
}
}
return result;
}
applyOrdering(results, orderBy) {
return results.sort((a, b) => {
for (const order of orderBy) {
const aVal = a[order.field];
const bVal = b[order.field];
let comparison = 0;
if (aVal < bVal)
comparison = -1;
else if (aVal > bVal)
comparison = 1;
if (comparison !== 0) {
return order.direction === 'desc' ? -comparison : comparison;
}
}
return 0;
});
}
generateId() {
return `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
calculateSize(data) {
try {
return JSON.stringify(data).length * 2; // Rough estimate (UTF-16)
}
catch {
return 1024; // Default size for non-serializable objects
}
}
getMemoryUsage() {
return this.metrics.totalSize;
}
async compress(data) {
if (!this.config.compressionEnabled) {
return Buffer.from(JSON.stringify(data));
}
const jsonString = JSON.stringify(data);
return new Promise((resolve, reject) => {
zlib.gzip(jsonString, (err, compressed) => {
if (err)
reject(err);
else
resolve(compressed);
});
});
}
async decompress(data) {
if (!this.config.compressionEnabled || !Buffer.isBuffer(data)) {
return data;
}
return new Promise((resolve, reject) => {
zlib.gunzip(data, (err, decompressed) => {
if (err)
reject(err);
else {
try {
resolve(JSON.parse(decompressed.toString()));
}
catch (parseErr) {
reject(parseErr);
}
}
});
});
}
updateIndexes(id, data) {
// Simple indexing for common fields
const indexableFields = ['type', 'category', 'status', 'name'];
for (const field of indexableFields) {
if (data[field] !== undefined) {
if (!this.indexes.has(field)) {
this.indexes.set(field, new Map());
}
const fieldIndex = this.indexes.get(field);
if (!fieldIndex.has(data[field])) {
fieldIndex.set(data[field], new Set());
}
fieldIndex.get(data[field]).add(id);
}
}
}
removeFromIndexes(id) {
for (const [_field, fieldIndex] of this.indexes) {
for (const [value, idSet] of fieldIndex) {
idSet.delete(id);
if (idSet.size === 0) {
fieldIndex.delete(value);
}
}
}
}
setupGarbageCollection() {
this.gcTimer = setInterval(() => {
this.runGarbageCollection().catch(error => {
logger_1.default.error('Garbage collection failed:', error);
});
}, 60000); // Run GC every minute
}
async runGarbageCollection() {
const memoryUsage = this.getMemoryUsage();
const threshold = this.config.maxMemoryMB * 1024 * 1024 * this.config.gcThreshold;
if (memoryUsage < threshold) {
return; // No need for GC
}
logger_1.default.debug('Running garbage collection');
const startTime = Date.now();
// Sort records by access patterns (LRU)
const records = Array.from(this.storage.entries()).sort(([, a], [, b]) => {
return a.accessed.getTime() - b.accessed.getTime();
});
// Remove least recently used records
const targetSize = this.config.maxMemoryMB * 1024 * 1024 * 0.7; // Target 70% usage
let currentSize = memoryUsage;
let removedCount = 0;
for (const [id, record] of records) {
if (currentSize <= targetSize)
break;
this.storage.delete(id);
this.removeFromIndexes(id);
currentSize -= record.size;
removedCount++;
}
// Update metrics
this.metrics.totalRecords -= removedCount;
this.metrics.totalSize = currentSize;
this.metrics.gcRuns++;
this.metrics.lastGc = new Date();
const gcTime = Date.now() - startTime;
logger_1.default.debug(`GC completed: removed ${removedCount} records in ${gcTime}ms`);
this.emit('garbage_collected', { removedCount, gcTime, memoryFreed: memoryUsage - currentSize });
}
setupPersistence() {
if (!this.config.persistenceFile)
return;
this.persistenceTimer = setInterval(() => {
this.saveToPersistence().catch(error => {
logger_1.default.error('Failed to save to persistence:', error);
});
}, 300000); // Save every 5 minutes
}
async loadFromPersistence() {
if (!this.config.persistenceFile)
return;
try {
const data = await fs_1.promises.readFile(this.config.persistenceFile, 'utf-8');
const persistedData = JSON.parse(data);
for (const recordData of persistedData.records || []) {
const record = {
...recordData,
created: new Date(recordData.created),
updated: new Date(recordData.updated),
accessed: new Date(recordData.accessed)
};
this.storage.set(record.id, record);
// Rebuild indexes
if (!record.compressed) {
this.updateIndexes(record.id, record.data);
}
}
// Update metrics
this.metrics.totalRecords = this.storage.size;
this.metrics.totalSize = Array.from(this.storage.values()).reduce((sum, record) => sum + record.size, 0);
logger_1.default.info(`Loaded ${this.storage.size} records from persistence`);
}
catch (error) {
if (error.code !== 'ENOENT') {
logger_1.default.error('Failed to load from persistence:', error);
}
}
}
async saveToPersistence() {
if (!this.config.persistenceFile)
return;
try {
const persistenceDir = path.dirname(this.config.persistenceFile);
await fs_1.promises.mkdir(persistenceDir, { recursive: true });
const persistedData = {
timestamp: new Date().toISOString(),
records: Array.from(this.storage.values())
};
await fs_1.promises.writeFile(this.config.persistenceFile, JSON.stringify(persistedData, null, 2), 'utf-8');
logger_1.default.debug(`Saved ${this.storage.size} records to persistence`);
}
catch (error) {
logger_1.default.error('Failed to save to persistence:', error);
}
}
updateMetrics(success, responseTime) {
this.metrics.totalQueries++;
this.metrics.totalResponseTime += responseTime;
if (success) {
this.metrics.successfulQueries++;
}
else {
this.metrics.failedQueries++;
}
}
}
exports.InMemoryStorageLayer = InMemoryStorageLayer;
exports.default = InMemoryStorageLayer;
//# sourceMappingURL=InMemoryStorageLayer.js.map