@yihuangdb/storage-object
Version:
A Node.js storage object layer library using Redis OM
296 lines • 10.9 kB
JavaScript
;
/**
* Redis OM Pattern Scanner
* Finds and manages all keys created by Redis OM for a schema
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.RedisOMPatternScanner = void 0;
exports.monitorAllSchemas = monitorAllSchemas;
exports.cleanupAllSchemas = cleanupAllSchemas;
class RedisOMPatternScanner {
entityName;
client;
// Redis OM's predictable patterns
REDIS_OM_PATTERNS = [
// Primary patterns
'{entity}:*', // Main entity keys
'idx:{entity}', // RediSearch index definition
'idx:{entity}:*', // RediSearch index data
// Field indexes
'{entity}#*', // Field index pattern
// FT Search patterns
'ft:{entity}', // FT index
'ft:{entity}:*', // FT index data
'ft.{entity}', // Alternative FT format
// Metadata patterns
'RedisOM:{entity}:*', // Redis OM metadata
'_meta:{entity}:*', // Custom metadata
'{entity}:index', // Index definition
'{entity}:index:*', // Index data
// Temporary/Lock patterns
'_tmp:{entity}:*', // Temporary keys
'_lock:{entity}:*', // Lock keys
// With prefix support
'*:{entity}:*' // Any prefix
];
constructor(entityName, client) {
this.entityName = entityName;
this.client = client;
}
/**
* Get all patterns for this entity
*/
getPatterns() {
return this.REDIS_OM_PATTERNS.map(pattern => pattern.replace('{entity}', this.entityName));
}
/**
* Get keys matching a pattern from registry
*/
async scanPattern(pattern) {
// Extract entity name from pattern if possible
// Patterns like: storage:entityName:*, idx:entityName:*
const parts = pattern.split(':');
if (parts.length >= 2) {
// Try to extract entity name
const entityName = parts[1];
if (entityName && !entityName.includes('*')) {
// Use cursor-based SCAN to get keys for this entity
const { getRedisKeyManager } = require('../redis-key-manager');
const keyManager = getRedisKeyManager();
const entityPattern = `${parts[0]}:${entityName}:*`;
return await keyManager.getAllKeysMatching(this.client, entityPattern);
}
}
// For other patterns, use cursor-based SCAN
const { getRedisKeyManager } = require('../redis-key-manager');
const keyManager = getRedisKeyManager();
return await keyManager.getAllKeysMatching(this.client, pattern);
}
/**
* Find all keys for this schema
*/
async findAllSchemaKeys() {
const patterns = this.getPatterns();
const results = new Map();
const breakdown = {};
let totalKeys = 0;
console.log(`Scanning for ${this.entityName} keys using ${patterns.length} patterns...`);
for (const pattern of patterns) {
try {
const keys = await this.scanPattern(pattern);
if (keys.length > 0) {
// Deduplicate keys
const uniqueKeys = Array.from(new Set(keys));
results.set(pattern, uniqueKeys);
breakdown[pattern] = uniqueKeys.length;
totalKeys += uniqueKeys.length;
console.log(` ✓ Found ${uniqueKeys.length} keys matching: ${pattern}`);
}
}
catch (error) {
console.error(` ✗ Error scanning pattern ${pattern}:`, error);
}
}
// Remove duplicates across patterns
const allKeys = new Set();
for (const keys of results.values()) {
keys.forEach(key => allKeys.add(key));
}
console.log(`Total unique keys found: ${allKeys.size}`);
return {
patterns: results,
totalKeys: allKeys.size,
breakdown
};
}
/**
* Delete all keys for this schema
*/
async deleteAllSchemaKeys() {
const scanResult = await this.findAllSchemaKeys();
let deleted = 0;
const errors = [];
const usedPatterns = [];
// Collect all unique keys
const allKeys = new Set();
for (const keys of scanResult.patterns.values()) {
keys.forEach(key => allKeys.add(key));
}
if (allKeys.size === 0) {
console.log(`No keys found for ${this.entityName}`);
return { deleted: 0, patterns: [], errors: [] };
}
console.log(`Deleting ${allKeys.size} keys for ${this.entityName}...`);
// Delete in batches
const keysArray = Array.from(allKeys);
const batchSize = 1000;
for (let i = 0; i < keysArray.length; i += batchSize) {
const batch = keysArray.slice(i, i + batchSize);
try {
// Use UNLINK for async deletion (non-blocking)
await this.client.unlink(batch);
deleted += batch.length;
if ((i + batchSize) % 10000 === 0) {
console.log(` Deleted ${deleted}/${allKeys.size} keys...`);
}
}
catch (error) {
const errorMsg = `Failed to delete batch at index ${i}: ${error}`;
errors.push(errorMsg);
console.error(errorMsg);
}
}
console.log(`✓ Successfully deleted ${deleted} keys`);
return {
deleted,
patterns: Array.from(scanResult.patterns.keys()),
errors
};
}
/**
* Get statistics about keys
*/
async getKeyStats() {
const result = await this.findAllSchemaKeys();
// Categorize keys by type
const byType = {
entities: 0,
indexes: 0,
search: 0,
metadata: 0,
temporary: 0,
other: 0
};
for (const [pattern, keys] of result.patterns) {
const count = keys.length;
if (pattern.startsWith(this.entityName + ':')) {
byType.entities += count;
}
else if (pattern.includes('idx') || pattern.includes('index')) {
byType.indexes += count;
}
else if (pattern.includes('ft')) {
byType.search += count;
}
else if (pattern.includes('meta') || pattern.includes('RedisOM')) {
byType.metadata += count;
}
else if (pattern.includes('tmp') || pattern.includes('lock')) {
byType.temporary += count;
}
else {
byType.other += count;
}
}
// Estimate memory usage (optional)
let memoryUsage;
try {
// Sample first 100 keys for memory estimation
const allKeys = Array.from(result.patterns.values()).flat();
const sampleKeys = allKeys.slice(0, Math.min(100, allKeys.length));
if (sampleKeys.length > 0) {
let totalMemory = 0;
for (const key of sampleKeys) {
try {
// @ts-ignore - memoryUsage might not be available
const usage = await this.client.memoryUsage(key);
totalMemory += usage || 0;
}
catch (e) {
// Memory command might not be available
}
}
// Extrapolate
if (totalMemory > 0) {
memoryUsage = Math.round(totalMemory * (allKeys.length / sampleKeys.length));
}
}
}
catch (error) {
console.warn('Could not estimate memory usage:', error);
}
return {
total: result.totalKeys,
byType,
memoryUsage
};
}
/**
* Find orphaned keys (keys that don't match expected patterns)
*/
async findOrphanedKeys() {
const orphaned = [];
// First, get all keys that contain the entity name
const allKeysWithEntity = await this.scanPattern(`*${this.entityName}*`);
// Get all keys from known patterns
const knownKeys = new Set();
const patterns = this.getPatterns();
for (const pattern of patterns) {
const keys = await this.scanPattern(pattern);
keys.forEach(key => knownKeys.add(key));
}
// Find keys that don't match any known pattern
for (const key of allKeysWithEntity) {
if (!knownKeys.has(key)) {
orphaned.push(key);
}
}
if (orphaned.length > 0) {
console.log(`Found ${orphaned.length} orphaned keys for ${this.entityName}`);
// Log sample of orphaned keys for debugging
console.log('Sample orphaned keys:', orphaned.slice(0, 5));
}
return orphaned;
}
/**
* Clean up orphaned keys
*/
async cleanupOrphans() {
const orphaned = await this.findOrphanedKeys();
if (orphaned.length === 0) {
return 0;
}
console.log(`Cleaning up ${orphaned.length} orphaned keys...`);
// Delete in batches
const batchSize = 1000;
for (let i = 0; i < orphaned.length; i += batchSize) {
const batch = orphaned.slice(i, i + batchSize);
await this.client.unlink(batch);
}
return orphaned.length;
}
}
exports.RedisOMPatternScanner = RedisOMPatternScanner;
/**
* Utility to monitor all schemas
*/
async function monitorAllSchemas(schemas, client) {
const report = {};
for (const schemaName of schemas) {
const scanner = new RedisOMPatternScanner(schemaName, client);
const stats = await scanner.getKeyStats();
report[schemaName] = stats;
// Log summary
console.log(`${schemaName}: ${stats.total} keys (${Object.entries(stats.byType)
.filter(([_, count]) => count > 0)
.map(([type, count]) => `${type}: ${count}`)
.join(', ')})`);
}
return report;
}
/**
* Clean up all schemas
*/
async function cleanupAllSchemas(schemas, client) {
const results = {};
for (const schemaName of schemas) {
console.log(`\nCleaning up ${schemaName}...`);
const scanner = new RedisOMPatternScanner(schemaName, client);
results[schemaName] = await scanner.deleteAllSchemaKeys();
}
// Summary
const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0);
console.log(`\nTotal keys deleted across all schemas: ${totalDeleted}`);
return results;
}
//# sourceMappingURL=redis-om-scanner.js.map