@yihuangdb/storage-object
Version:
A Node.js storage object layer library using Redis OM
231 lines • 9.16 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.PatternScanner = void 0;
const redis_key_manager_1 = require("../redis-key-manager");
class PatternScanner {
static keyManager = (0, redis_key_manager_1.getRedisKeyManager)();
/**
* Get patterns for a specific entity
*/
static getPatterns(entityName) {
// Use RedisKeyManager to get entity patterns
return this.keyManager.getEntityPatterns(entityName);
}
/**
* Scan all patterns and return keys grouped by pattern
*/
static async scanAllPatterns(client, entityName) {
const results = new Map();
if (entityName) {
// Entity-specific patterns
const patterns = this.getPatterns(entityName);
for (const pattern of patterns) {
const keys = await this.scanPattern(client, pattern);
results.set(pattern, keys);
}
}
else {
// Global patterns - use cursor-based SCAN to get keys
// Get all schema keys
const schemaKeys = await this.keyManager.getAllKeysMatching(client, 'schema:*');
const schemaMetaKeys = schemaKeys.filter(k => k.match(/^schema:[^:]+$/));
results.set('schema:*', schemaMetaKeys);
// Get all HLL keys
const hllKeys = await this.keyManager.getAllKeysMatching(client, 'schema:*:hll');
results.set('schema:*:hll', hllKeys);
// Get all version keys
const versionKeys = await this.keyManager.getAllKeysMatching(client, 'schema:*:versions');
results.set('schema:*:versions', versionKeys);
// Get all entity keys (need to scan for multiple patterns)
let entityKeys = [];
// Common entity patterns - you may need to adjust based on your entity names
const entityPatterns = ['*:*'];
for (const pattern of entityPatterns) {
const keys = await this.keyManager.getAllKeysMatching(client, pattern);
// Filter out schema, idx, and other non-entity keys
const filtered = keys.filter(k => !k.startsWith('schema:') && !k.startsWith('idx:') && !k.startsWith('registry:') && !k.startsWith('_'));
// Use concat to safely handle large arrays without stack overflow
entityKeys = entityKeys.concat(filtered);
}
results.set('*:*', entityKeys);
// Get index keys if any
const indexKeys = await this.keyManager.getAllKeysMatching(client, 'idx:*');
results.set('idx:*', indexKeys);
}
return results;
}
/**
* Get keys using cursor-based SCAN
*/
static async scanPattern(client, pattern) {
// Use cursor-based SCAN from RedisKeyManager
return await this.keyManager.getAllKeysMatching(client, pattern);
// This forces proper key tracking through the registry
return [];
}
/**
* Extract entity name from a pattern
*/
static extractEntityFromPattern(pattern) {
// Common patterns:
// storage:entityName:*
// schema:entityName:*
// idx:entityName:*
// entityName:*
const parts = pattern.split(':');
if (parts.length >= 2) {
// For patterns like storage:entityName:* or schema:entityName:*
if (parts[0] === 'storage' || parts[0] === 'schema' || parts[0] === 'idx') {
const entityName = parts[1];
if (entityName && !entityName.includes('*')) {
return entityName;
}
}
else if (parts.length === 2 && parts[1] === '*') {
// For patterns like entityName:*
return parts[0];
}
}
return null;
}
/**
* Delete keys matching a pattern (using UNLINK for non-blocking)
*/
static async deleteByPattern(client, pattern) {
const keys = await this.scanPattern(client, pattern);
if (keys.length === 0) {
return 0;
}
// Use UNLINK for non-blocking deletion
const deleted = await client.unlink(keys);
return deleted;
}
/**
* Find orphaned keys not associated with registered schemas
*/
static async findOrphans(client) {
const orphans = [];
// Get registered schemas from registry using validated key
const registryKey = this.keyManager.getRegistrySchemasKey();
const registryData = await this.keyManager.safeHGetAll(client, registryKey);
const registeredSchemas = new Set(Object.keys(registryData));
// Scan for all Redis OM patterns
const patterns = [
'*:*', // All entity data patterns
'idx:*', // All index patterns
'ft:*', // Full-text search patterns
'RedisOM:*' // RedisOM metadata patterns
];
for (const pattern of patterns) {
const keys = await this.scanPattern(client, pattern);
for (const key of keys) {
// Extract schema name from key
const schemaName = this.extractSchemaName(key);
// Check if schema is registered
if (schemaName && !registeredSchemas.has(schemaName)) {
orphans.push(key);
}
}
}
return orphans;
}
/**
* Extract schema/entity name from a Redis key using RedisKeyManager
*/
static extractSchemaName(key) {
// Pattern: schema:name:suffix
if (key.startsWith('schema:')) {
const parts = key.split(':');
return parts[1] || null;
}
// Pattern: entityName:id
if (key.includes(':') && !key.startsWith('idx:') && !key.startsWith('ft:')) {
const parts = key.split(':');
return parts[0] || null;
}
// Pattern: idx:entityName:field
if (key.startsWith('idx:')) {
const parts = key.split(':');
return parts[1] || null;
}
// Pattern: ft:entityName:info
if (key.startsWith('ft:')) {
const parts = key.split(':');
return parts[1] || null;
}
// Pattern: RedisOM:entityName:...
if (key.startsWith('RedisOM:')) {
const parts = key.split(':');
return parts[1] || null;
}
return null;
}
/**
* Delete all keys for a specific entity/schema
*/
static async deleteEntityKeys(client, entityName) {
// Get patterns from RedisKeyManager
const patterns = this.keyManager.getEntityPatterns(entityName);
let totalDeleted = 0;
for (const pattern of patterns) {
const deleted = await this.deleteByPattern(client, pattern);
totalDeleted += deleted;
}
return totalDeleted;
}
/**
* Scan for entity-specific patterns
*/
static async scanEntityPatterns(client, entityName) {
const results = new Map();
// Replace placeholders with actual entity name using the key manager
const entityPatterns = [
this.keyManager.getSchemaMetadataKey(entityName), // Schema metadata hash
this.keyManager.getSchemaHllKey(entityName), // HyperLogLog counter
this.keyManager.getSchemaVersionsKey(entityName), // Version history
this.keyManager.generateKey('{prefix}:{entityName}:*', { prefix: 'storage', entityName }), // Entity data
this.keyManager.generateKey('idx:{entityName}*', { entityName }) // Indexes
];
for (const pattern of entityPatterns) {
// For exact keys (no wildcards), check existence directly
if (!pattern.includes('*')) {
const exists = await client.exists(pattern);
results.set(pattern, exists > 0 ? [pattern] : []);
}
else {
// For patterns with wildcards, use SCAN
const keys = await this.scanPattern(client, pattern);
results.set(pattern, keys);
}
}
return results;
}
/**
* Get statistics about key distribution
*/
static async getKeyStatistics(client) {
// Get pattern statistics
const allPatterns = await this.scanAllPatterns(client);
const byPattern = {};
const byEntity = {};
let total = 0;
for (const [pattern, keys] of allPatterns) {
byPattern[pattern] = keys.length;
total += keys.length;
// Extract entity names from keys
for (const key of keys) {
const entityName = this.extractEntityFromPattern(key);
if (entityName) {
byEntity[entityName] = (byEntity[entityName] || 0) + 1;
}
}
}
return {
total,
byPattern,
byEntity
};
}
}
exports.PatternScanner = PatternScanner;
//# sourceMappingURL=pattern-scanner.js.map