@yihuangdb/storage-object
Version:
A Node.js storage object layer library using Redis OM
1,213 lines • 46.2 kB
JavaScript
"use strict";
/**
* @module storage-system
* @description Central management system for all storage instances.
* Provides a unified, object-oriented API for creating, managing, and monitoring storage objects.
*
* @since 0.1.0
* @author StorageObject Team
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.StorageSystem = void 0;
const storage_1 = require("./storage");
const storage_schema_1 = require("./storage-schema");
const schema_registry_1 = require("./schema-registry");
const redis_om_version_validator_1 = require("./redis-om-version-validator");
const redis_key_manager_1 = require("./redis-key-manager");
const export_import_manager_1 = require("./export-import-manager");
const storage_version_manager_1 = require("./storage-version-manager");
const connection_1 = require("./connection");
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
/**
* StorageSystem - Central management class for all storage operations.
*
* This class provides static methods for managing storage instances, schemas, and system-wide operations.
* All methods are static - no instantiation is required or allowed.
*
* @class StorageSystem
* @since 0.1.0
*
* @example Basic Usage
* ```typescript
* // Initialize the system with configuration
* await StorageSystem.initialize({
* redis: { host: 'localhost', port: 6379 },
* connectionPool: { maxSize: 20 }
* });
*
* // Create a new storage
* const userStorage = await StorageSystem.create('users', schema);
*
* // Get existing storage
* const storage = await StorageSystem.get('users');
*
* // List all storages
* const schemas = await StorageSystem.list();
*
* // Delete a storage
* await StorageSystem.delete('users');
* ```
*
* @example Advanced Operations
* ```typescript
* // Get or create a storage
* const storage = await StorageSystem.getOrCreate('products', schema);
*
* // Check if storage exists
* if (await StorageSystem.exists('orders')) {
* const orders = await StorageSystem.get('orders');
* }
*
* // Get storage statistics
* const stats = await StorageSystem.getStats('users');
* console.log('Total users:', stats.objectCount);
*
* // Backup and restore
* const backup = await StorageSystem.backup({ includeData: true });
* await StorageSystem.restore(backup, { overwrite: true });
* ```
*/
class StorageSystem {
/** Singleton schema registry instance */
static registry = null;
/** System configuration */
static config = {};
/** Whether the system has been initialized */
static initialized = false;
/** Storage version managers for each schema */
static storageVersionManagers = new Map();
/** Export/import managers for each schema */
static exportImportManagers = new Map();
/** Storage instances for system-wide operations */
static storageInstances = new Map();
/** Version validator instance */
static versionValidator = null;
/**
* Private constructor to prevent instantiation
*/
constructor() {
// Private constructor to enforce static-only usage
}
/**
* Get or create the schema registry instance
*/
static getRegistry() {
if (!this.registry) {
this.registry = (0, schema_registry_1.getSchemaRegistry)();
}
return this.registry;
}
/**
* Initialize the StorageSystem with global configuration.
* This method is optional - StorageSystem works with defaults if not called.
*
* @param config - System configuration options
* @returns Promise that resolves when initialization is complete
*
* @example
* ```typescript
* // Optional initialization with custom config
* await StorageSystem.initialize({
* redis: { host: 'localhost', port: 6379 },
* defaultOptions: {
* useJSON: true,
* enableOptimisticLocking: true
* }
* });
*
* // Or just start using it with defaults
* const users = await StorageSystem.create('users', schema);
* ```
*/
static async initialize(config) {
if (config) {
this.config = { ...this.config, ...config };
}
this.initialized = true;
// Apply any global configuration to the registry
const registry = this.getRegistry();
// Note: Registry initialization logic would go here if needed
}
/**
* Configure StorageSystem with a config object or function.
* Alternative to initialize() with a more flexible API.
*
* @param configOrFn - Configuration object or function that returns config
* @returns The StorageSystem class for chaining
*
* @example
* ```typescript
* // Configure with object
* StorageSystem.configure({
* redis: { host: 'redis.example.com' }
* });
*
* // Configure with function
* StorageSystem.configure((config) => {
* config.redis = { host: process.env.REDIS_HOST };
* config.defaultOptions = { useJSON: true };
* });
* ```
*/
static configure(configOrFn) {
if (typeof configOrFn === 'function') {
configOrFn(this.config);
}
else {
this.config = { ...this.config, ...configOrFn };
}
this.initialized = true;
return this;
}
/**
* Create a new storage instance with the given schema.
*
* This is the primary method for creating storage instances. It automatically:
* - Validates the schema
* - Registers it with the schema registry
* - Initializes the storage with optimized settings
* - Returns a ready-to-use StorageObject
*
* @template T - Type of the entities to be stored
* @param name - Unique name for the storage
* @param schema - Schema configuration or StorageSchema instance
* @param options - Optional storage configuration
* @returns Promise resolving to the created StorageObject instance
*
* @example Simple Usage
* ```typescript
* // Using inline schema definition
* const users = await StorageSystem.create('users', {
* name: 'text',
* email: 'string',
* age: 'number'
* });
*
* // Using StorageSchema
* const schema = StorageSchema.define({
* name: { type: 'text', indexed: true },
* email: { type: 'string', indexed: true, required: true },
* age: { type: 'number', indexed: true }
* });
* const users = await StorageSystem.create('users', schema);
* ```
*
* @example Advanced Usage
* ```typescript
* const users = await StorageSystem.create('users', {
* email: {
* type: 'string',
* indexed: true,
* required: true,
* validate: (value) => value.includes('@') || 'Invalid email'
* },
* role: {
* type: 'string',
* indexed: true,
* default: 'user'
* },
* tags: { type: 'string[]', separator: ',', indexed: true }
* }, {
* enableOptimisticLocking: true,
* enableChangeTracking: true
* });
* ```
*/
static async create(name, schema, options) {
// Handle both SchemaConfig and StorageSchema instances
let schemaConfig;
if (schema instanceof storage_schema_1.StorageSchema) {
schemaConfig = schema.getFields();
}
else {
schemaConfig = schema;
}
// Handle both SchemaConfig and StorageSchema instances (continuation)
// Check if version validation is enabled (default: true)
const versionValidationEnabled = this.config.versionValidation?.enabled !== false;
if (versionValidationEnabled) {
// Initialize and validate Redis OM version
const strictMode = this.config.versionValidation?.strictMode !== false;
const versionValidator = (0, redis_om_version_validator_1.getRedisOMVersionValidator)(strictMode);
const connection = connection_1.RedisConnection.getInstance(options);
const client = await connection.connect();
await versionValidator.initialize(client);
const versionValidation = await versionValidator.validateVersion();
if (!versionValidation.valid) {
const errorMessage = `Redis OM version validation failed: ${versionValidation.errors.join(', ')}`;
if (versionValidation.warnings.length > 0) {
console.warn('Version warnings:', versionValidation.warnings.join(', '));
}
throw new Error(errorMessage);
}
// Store validator instance for future use
this.versionValidator = versionValidator;
}
const registry = this.getRegistry();
// Merge with default options if configured
// Enable change tracking by default unless explicitly disabled
const mergedOptions = {
enableChangeTracking: true, // Default to true
...this.config.defaultOptions,
...options,
};
// Register the schema
await registry.register(name, schemaConfig, mergedOptions);
// Create and initialize the storage
const storage = new storage_1.StorageObject(name, schemaConfig, mergedOptions);
await storage.initialize();
// Register the storage instance with the registry to enable statistics tracking
await registry.registerStorageInstance(name, storage);
// Register for system-wide operations
this.storageInstances.set(name, storage);
// Get the schema version manager from registry and set it on the storage
const schemaVersionManager = registry.getVersionManager(name);
if (schemaVersionManager) {
storage.setSchemaVersionManager(schemaVersionManager);
}
// Initialize version tracking (enabled by default, skip only if explicitly disabled)
if (mergedOptions.enableChangeTracking !== false) {
const prefix = mergedOptions.prefix || 'storage';
const keyManager = (0, redis_key_manager_1.getRedisKeyManager)();
const redis = storage.getRedis();
const storageVersionManager = new storage_version_manager_1.StorageVersionManager(keyManager, redis);
const exportImportManager = new export_import_manager_1.ExportImportManager(keyManager, storageVersionManager, redis);
this.storageVersionManagers.set(name, storageVersionManager);
this.exportImportManagers.set(name, exportImportManager);
// Enable change tracking on the storage instance
storage.enableChangeTracking(storageVersionManager);
}
return storage;
}
/**
* Get an existing storage instance by name.
*
* @param name - Name of the storage to retrieve
* @returns Promise resolving to the StorageObject instance or null if not found
*
* @example
* ```typescript
* const users = await StorageSystem.get('users');
* if (users) {
* const allUsers = await users.findAll();
* }
* ```
*/
static async get(name) {
const registry = this.getRegistry();
return registry.getStorage(name);
}
/**
* Open an existing storage or throw if not found.
* Convenience method that ensures the storage exists.
*
* @param name - Name of the storage to open
* @returns Promise resolving to the StorageObject instance
* @throws Error if storage doesn't exist
*
* @example
* ```typescript
* const users = await StorageSystem.open('users');
* // No null check needed - throws if not found
* const allUsers = await users.findAll();
* ```
*/
static async open(name) {
const storage = await this.get(name);
if (!storage) {
throw new Error(`Storage '${name}' not found. Use StorageSystem.create() to create it first.`);
}
return storage;
}
/**
* Delete a storage and all its data.
*
* @param name - Name of the storage to delete
* @returns Promise resolving to true if deleted, false otherwise
*
* @example
* ```typescript
* const deleted = await StorageSystem.delete('users');
* console.log(deleted ? 'Deleted successfully' : 'Not found');
* ```
*/
static async delete(name) {
const registry = this.getRegistry();
// Use the new unregister method that properly cleans up cache
return await registry.unregister(name);
}
/**
* Clear all data from a storage without deleting the schema.
*
* @param name - Name of the storage to clear
* @returns Promise resolving when cleared
* @throws Error if storage doesn't exist
*
* @example
* ```typescript
* await StorageSystem.clear('users');
* console.log('All user data cleared');
* ```
*/
static async clear(name) {
const storage = await this.open(name);
await storage.clear();
}
/**
* Get storage metadata including statistics
*
* @param name - Name of the storage
* @returns Promise resolving to metadata or null if not found
*
* @example
* ```typescript
* const metadata = await StorageSystem.getMetadata('users');
* if (metadata) {
* console.log('Object count:', metadata.statistics.objectCount);
* }
* ```
*/
static async getMetadata(name) {
const registry = this.getRegistry();
return registry.getSchemaMetadata(name);
}
/**
* Validate a storage schema against its data
*
* @param name - Name of the storage to validate
* @returns Promise resolving to validation results
*
* @example
* ```typescript
* const validation = await StorageSystem.validate('users');
* if (!validation.valid) {
* console.error('Validation errors:', validation.errors);
* }
* ```
*/
static async validate(name) {
const registry = this.getRegistry();
return registry.validate(name);
}
/**
* Check if a storage exists.
*
* @param name - Name of the storage to check
* @returns Promise resolving to true if storage exists, false otherwise
*
* @example
* ```typescript
* if (await StorageSystem.exists('users')) {
* const users = await StorageSystem.open('users');
* }
* ```
*/
static async exists(name) {
const registry = this.getRegistry();
const metadata = await registry.getSchemaMetadata(name);
return metadata !== null;
}
/**
* Get all storage names.
*
* @returns Promise resolving to array of storage names
*
* @example
* ```typescript
* const names = await StorageSystem.names();
* console.log('Available storages:', names);
* // ['users', 'products', 'orders']
* ```
*/
static async names() {
const schemas = await this.list();
return schemas.map(s => s.name);
}
/**
* List all registered storage schemas with metadata.
*
* @returns Promise resolving to array of schema metadata
*
* @example
* ```typescript
* const schemas = await StorageSystem.list();
* schemas.forEach(schema => {
* console.log(`${schema.name}: ${schema.statistics.objectCount} objects`);
* });
* ```
*/
static async list() {
const registry = this.getRegistry();
return registry.getAllSchemas();
}
/**
* Flush all Redis data (DANGEROUS - requires confirmation)
*
* @param confirmation - Must be 'CONFIRM_FLUSH_ALL' to execute
* @returns Promise resolving to true if flushed, false if not confirmed
*
* @example
* ```typescript
* // This will delete ALL Redis data - use with extreme caution!
* const flushed = await StorageSystem.flushAll('CONFIRM_FLUSH_ALL');
* ```
*/
static async flushAll(confirmation) {
if (confirmation !== 'CONFIRM_FLUSH_ALL') {
console.warn('StorageSystem.flushAll() requires confirmation string "CONFIRM_FLUSH_ALL"');
return false;
}
const registry = this.getRegistry();
// Clear all persisted schemas
await registry.clearAllPersistedSchemas();
// Get Redis connection to flush all data
console.warn('FLUSHING ALL REDIS DATA');
try {
const connection = connection_1.RedisConnection.getInstance();
const client = connection.getClient();
// Execute FLUSHALL command to clear all Redis data
await client.flushAll();
console.warn('Redis FLUSHALL completed successfully');
return true;
}
catch (error) {
console.error('Failed to flush Redis data:', error);
// If we can't get connection, try to continue anyway
// as the storage instances might not be initialized yet
return true;
}
}
/**
* Get or create a storage instance.
*
* @param name - Name of the storage
* @param schema - Schema configuration (required if storage doesn't exist)
* @param options - Optional storage configuration
* @returns Promise resolving to the StorageObject instance
*
* @example
* ```typescript
* // Will create if doesn't exist, or return existing
* const users = await StorageSystem.getOrCreate('users', {
* name: 'text',
* email: 'string'
* });
* ```
*/
static async getOrCreate(name, schema, options) {
// Try to get existing storage
let storage = await this.get(name);
if (storage) {
return storage;
}
// Create new storage if schema is provided
if (!schema) {
throw new Error(`Storage '${name}' not found and no schema provided`);
}
return this.create(name, schema, options);
}
/**
* Get storage statistics
*
* @param name - Name of the storage
* @returns Promise resolving to statistics or null if not found
*
* @example
* ```typescript
* const stats = await StorageSystem.getStats('users');
* if (stats) {
* console.log('Average create time:', stats.performance.averageCreateTime);
* }
* ```
*/
static async getStats(name) {
const metadata = await this.getMetadata(name);
if (!metadata) {
return null;
}
return {
objectCount: metadata.statistics.objectCount,
totalOperations: metadata.statistics.totalOperations,
averageObjectSize: metadata.statistics.averageObjectSize,
performance: {
averageCreateTime: metadata.performance.averageCreateTime,
averageReadTime: metadata.performance.averageReadTime,
averageUpdateTime: metadata.performance.averageUpdateTime,
averageDeleteTime: metadata.performance.averageDeleteTime,
},
};
}
/**
* Backup all storages or specific storages
*
* @param options - Backup options
* @returns Promise resolving to backup JSON string
*
* @example
* ```typescript
* const backup = await StorageSystem.backup({
* names: ['users', 'products'],
* includeData: true
* });
* ```
*/
static async backup(options) {
const registry = this.getRegistry();
if (options?.names) {
// Backup specific storages
const backup = {
version: '1.0.0',
timestamp: new Date(),
schemas: {},
data: {},
};
for (const name of options.names) {
const metadata = await registry.getSchemaMetadata(name);
if (metadata) {
backup.schemas[name] = metadata;
if (options.includeData) {
const storage = await registry.getStorage(name);
if (storage) {
backup.data[name] = await storage.findAll();
}
}
}
}
return JSON.stringify(backup, null, 2);
}
// Backup all storages
return registry.backup({
includeData: options?.includeData,
compress: options?.compress,
});
}
/**
* Restore storages from backup
*
* @param backupJson - Backup JSON string
* @param options - Restore options
* @returns Promise resolving to restore results
*
* @example
* ```typescript
* const result = await StorageSystem.restore(backupJson, {
* overwrite: true,
* validateSchema: true
* });
* ```
*/
static async restore(backupJson, options) {
const registry = this.getRegistry();
return registry.restore(backupJson, options);
}
/**
* Clean up inactive storages
*
* @param inactiveDays - Number of days of inactivity before cleanup (default: 30)
* @returns Promise resolving to cleanup results
*
* @example
* ```typescript
* const result = await StorageSystem.cleanup(90);
* console.log('Cleaned:', result.cleaned);
* console.log('Kept:', result.kept);
* ```
*/
static async cleanup(inactiveDays = 30) {
const registry = this.getRegistry();
return registry.cleanup(inactiveDays);
}
/**
* Get global statistics across all storages
*
* @returns Promise resolving to global statistics
*
* @example
* ```typescript
* const stats = await StorageSystem.getGlobalStats();
* console.log('Total schemas:', stats.totalSchemas);
* console.log('Total objects:', stats.totalObjects);
* ```
*/
static async getGlobalStats() {
const registry = this.getRegistry();
return registry.getStatistics();
}
/**
* Cleanup orphaned Redis OM keys not managed by current schemas
* Scans using Redis OM patterns and removes keys not in registry
*
* @param schemaName - Optional specific schema to clean (all if not provided)
* @returns Promise resolving to cleanup results
*
* @example
* ```typescript
* // Clean all orphaned keys
* const result = await StorageSystem.cleanupOrphanedKeys();
* console.log(`Deleted ${result.deleted} orphaned keys`);
*
* // Clean specific schema
* const result = await StorageSystem.cleanupOrphanedKeys('users');
* ```
*/
static async cleanupOrphanedKeys(schemaName) {
const { PatternScanner } = await Promise.resolve().then(() => __importStar(require('./utils/pattern-scanner')));
const { createRedisClient } = await Promise.resolve().then(() => __importStar(require('./utils/redis-client')));
// Create a client for scanning
const client = createRedisClient();
await client.connect();
const result = {
deleted: 0,
patterns: [],
orphans: []
};
try {
if (schemaName) {
// Clean specific schema
const registeredSchemas = await client.hGetAll('registry:schemas');
const isRegistered = schemaName in registeredSchemas;
if (!isRegistered) {
// Schema not registered - all its keys are orphans
// First, scan for all keys to populate orphans array
const keyManager = (0, redis_key_manager_1.getRedisKeyManager)();
const patterns = [
keyManager.generateKey('{entityName}:*', { entityName: schemaName }),
keyManager.generateKey('idx:{entityName}*', { entityName: schemaName }),
keyManager.generateKey('ft:{entityName}*', { entityName: schemaName }),
keyManager.generateKey('RedisOM:{entityName}:*', { entityName: schemaName }),
keyManager.generateKey('schema:{entityName}*', { entityName: schemaName })
];
for (const pattern of patterns) {
const keys = await PatternScanner.scanPattern(client, pattern);
result.orphans.push(...keys);
}
// Then delete them
const deleted = await PatternScanner.deleteEntityKeys(client, schemaName);
result.deleted = deleted;
result.patterns = patterns;
}
}
else {
// Find all orphaned keys
const orphanKeys = await PatternScanner.findOrphans(client);
result.orphans = orphanKeys;
// Delete orphaned keys in batches
if (orphanKeys.length > 0) {
const batchSize = 1000;
for (let i = 0; i < orphanKeys.length; i += batchSize) {
const batch = orphanKeys.slice(i, i + batchSize);
const deleted = await client.unlink(batch);
result.deleted += deleted;
}
}
// Patterns used for scanning
result.patterns = [
'*:*',
'idx:*',
'ft:*',
'RedisOM:*',
'schema:*'
];
}
return result;
}
finally {
// Always close the client
await client.quit();
}
}
/**
* Monitor storage changes with a callback
*
* @param name - Name of the storage to monitor
* @param callback - Callback function for events
* @returns Unsubscribe function
*
* @example
* ```typescript
* const unsubscribe = StorageSystem.monitor('users', (event) => {
* console.log(`${event.type} operation on ${event.entityId}`);
* });
*
* // Later: unsubscribe();
* ```
*/
static monitor(name, callback) {
// This would require implementing event emitters in StorageObject
// For now, returning a no-op unsubscribe function
console.log(`Monitoring for ${name} not yet implemented`);
return () => { };
}
/**
* Migrate data from one schema version to another
*
* @param name - Name of the storage to migrate
* @param newSchema - New schema configuration
* @param options - Migration options
* @returns Promise resolving to migration results
*
* @example
* ```typescript
* const result = await StorageSystem.migrate('users', newSchema, {
* transform: (oldData) => ({
* ...oldData,
* fullName: `${oldData.firstName} ${oldData.lastName}`
* })
* });
* ```
*/
static async migrate(name, newSchema, options) {
const result = {
success: true,
migrated: 0,
failed: 0,
errors: [],
};
try {
const storage = await this.get(name);
if (!storage) {
throw new Error(`Storage ${name} not found`);
}
// Get all existing data
const allData = await storage.findAll();
// Create new storage with new schema
const tempName = `${name}_migration_${Date.now()}`;
const newStorage = await this.create(tempName, newSchema, {
prefix: `migration_${name}`,
});
// Migrate data in batches
const batchSize = options?.batchSize || 100;
for (let i = 0; i < allData.length; i += batchSize) {
const batch = allData.slice(i, i + batchSize);
for (const item of batch) {
try {
const { entityId, ...data } = item;
const transformedData = options?.transform ? options.transform(data) : data;
// Validate if requested
if (options?.validateData) {
// Use the new schema's validation if available
const { StorageSchema } = await Promise.resolve().then(() => __importStar(require('./storage-schema')));
const schemaValidator = StorageSchema.define(newSchema);
const validationResult = schemaValidator.validate(transformedData);
if (!validationResult.valid) {
throw new Error(`Validation failed: ${validationResult.errors.join(', ')}`);
}
}
await newStorage.create(transformedData);
result.migrated++;
}
catch (error) {
result.failed++;
result.errors.push(error instanceof Error ? error.message : String(error));
}
}
}
if (result.failed === 0) {
// Clear old storage and copy data back
await storage.clear();
// Update schema in registry
const registry = this.getRegistry();
await registry.register(name, newSchema);
// Copy data back
const migratedData = await newStorage.findAll();
for (const item of migratedData) {
const { entityId, ...data } = item;
await storage.create(data);
}
// Clean up temp storage
await newStorage.clear();
await newStorage.disconnect();
}
else {
result.success = false;
}
}
catch (error) {
result.success = false;
result.errors.push(error instanceof Error ? error.message : String(error));
}
return result;
}
/**
* Export all storage data and schemas to a directory
*
* @param outputDir - Directory to export to
* @param options - Export options
* @returns Export result with metadata
*/
static async exportAll(outputDir, options) {
const systemExportId = `system-export-${Date.now()}-${Math.random().toString(36).substring(7)}`;
const exportTimestamp = Date.now();
await fs.promises.mkdir(outputDir, { recursive: true });
const schemasToExport = options?.schemasToExport || Array.from(this.storageInstances.keys());
const exportedSchemas = [];
let totalEntities = 0;
for (const schemaName of schemasToExport) {
const storage = this.storageInstances.get(schemaName);
const manager = this.exportImportManagers.get(schemaName);
if (!storage || !manager) {
console.warn(`Schema ${schemaName} not found, skipping...`);
continue;
}
const schemaOutputDir = path.join(outputDir, schemaName);
await fs.promises.mkdir(schemaOutputDir, { recursive: true });
const filePath = path.join(schemaOutputDir, `${schemaName}-export`);
const metadata = await manager.exportFull(storage, filePath, {
exportFormat: options?.exportFormat,
compressOutput: options?.compressOutput,
includeSchema: options?.includeSchema,
includeData: options?.includeData
});
exportedSchemas.push({
schemaName,
schemaVersionNum: metadata.exportedSchemaVersion,
storageVersionNum: metadata.exportedStorageVersion,
entityCount: metadata.exportedEntityCount,
exportFilePath: filePath
});
totalEntities += metadata.exportedEntityCount;
}
const manifest = {
systemExportId,
exportTimestamp,
exportedSchemas,
totalExportedEntities: totalEntities,
exportFormat: options?.exportFormat || 'json',
isCompressed: options?.compressOutput ?? true
};
await fs.promises.writeFile(path.join(outputDir, 'system-manifest.json'), JSON.stringify(manifest, null, 2));
return manifest;
}
/**
* Import all storage data and schemas from a directory
*
* @param inputDir - Directory to import from
* @param options - Import options
* @returns Import result with statistics
*/
static async importAll(inputDir, options) {
const systemImportId = `system-import-${Date.now()}-${Math.random().toString(36).substring(7)}`;
const startTime = Date.now();
// Read system manifest
const manifestPath = path.join(inputDir, 'system-manifest.json');
const manifestStr = await fs.promises.readFile(manifestPath, 'utf-8');
const manifest = JSON.parse(manifestStr);
// Get schemas to import
const schemasToImport = options?.schemasToImport || manifest.exportedSchemas.map(s => s.schemaName);
const importResults = [];
let totalImported = 0;
let totalFailed = 0;
// Import each schema
for (const schemaExport of manifest.exportedSchemas) {
if (!schemasToImport.includes(schemaExport.schemaName)) {
continue;
}
const storage = this.storageInstances.get(schemaExport.schemaName);
const manager = this.exportImportManagers.get(schemaExport.schemaName);
if (!storage || !manager) {
console.warn(`Schema ${schemaExport.schemaName} not found, skipping...`);
continue;
}
const schemaInputDir = path.join(inputDir, schemaExport.schemaName);
const filePath = schemaExport.exportFilePath;
const result = await manager.importFull(storage, filePath, options);
importResults.push({
schemaName: schemaExport.schemaName,
importedEntities: result.importedEntityCount,
failedEntities: result.failedEntityCount,
schemaUpdated: result.schemaWasUpdated,
errors: result.importErrors
});
totalImported += result.importedEntityCount;
totalFailed += result.failedEntityCount;
}
return {
systemImportId,
importTimestamp: Date.now(),
importedSchemas: importResults,
totalImportedEntities: totalImported,
totalFailedEntities: totalFailed,
totalImportDuration: Date.now() - startTime
};
}
/**
* Export incremental changes since last version
*/
static async exportIncremental(outputDir, lastExportedStorageVersions, options) {
const incrementalExportId = `incremental-${Date.now()}-${Math.random().toString(36).substring(7)}`;
const exportTimestamp = Date.now();
await fs.promises.mkdir(outputDir, { recursive: true });
const schemasToExport = options?.schemasToExport || Array.from(this.storageInstances.keys());
const exportedSchemas = [];
let totalChanges = 0;
for (const schemaName of schemasToExport) {
const storage = this.storageInstances.get(schemaName);
const manager = this.exportImportManagers.get(schemaName);
const versionManager = this.storageVersionManagers.get(schemaName);
if (!storage || !manager || !versionManager)
continue;
const lastStorageVersion = lastExportedStorageVersions.get(schemaName) || 0;
const currentStorageVersion = await versionManager.getCurrentStorageVersion();
if (lastStorageVersion >= currentStorageVersion) {
continue; // No changes for this schema
}
const schemaOutputDir = path.join(outputDir, schemaName);
await fs.promises.mkdir(schemaOutputDir, { recursive: true });
const filePath = path.join(schemaOutputDir, `${schemaName}-incremental`);
const metadata = await manager.exportFull(storage, filePath, {
...options,
incrementalExport: true,
fromStorageVersion: lastStorageVersion,
toStorageVersion: currentStorageVersion
});
exportedSchemas.push({
schemaName,
schemaVersionNum: metadata.exportedSchemaVersion,
storageVersionNum: metadata.exportedStorageVersion,
entityCount: metadata.exportedEntityCount,
exportFilePath: filePath
});
totalChanges += metadata.exportedEntityCount;
}
const manifest = {
systemExportId: incrementalExportId,
exportTimestamp,
exportedSchemas,
totalExportedEntities: totalChanges,
exportFormat: options?.exportFormat || 'json',
isCompressed: options?.compressOutput ?? true
};
await fs.promises.writeFile(path.join(outputDir, 'incremental-manifest.json'), JSON.stringify(manifest, null, 2));
return manifest;
}
/**
* System-wide backup
*/
static async systemBackup(backupName) {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const name = backupName || `system-backup-${timestamp}`;
const backupDir = path.join('./backups', name);
const result = await this.exportAll(backupDir, {
compressOutput: true,
includeSchema: true,
includeData: true
});
console.log(`Backup created: ${backupDir}`);
console.log(`- Schemas: ${result.exportedSchemas.length}`);
console.log(`- Total entities: ${result.totalExportedEntities}`);
return backupDir;
}
/**
* System-wide restore
*/
static async systemRestore(backupPath) {
return await this.importAll(backupPath, {
validateSchemaVersion: true,
entityMergeStrategy: 'replace',
continueOnError: false
});
}
/**
* Get system version status
*/
static async getSystemVersionStatus() {
const schemas = [];
let totalEntities = 0;
for (const [schemaName, storage] of this.storageInstances) {
const versionManager = this.storageVersionManagers.get(schemaName);
if (!versionManager)
continue;
const [storageVer, schemaVer, count] = await Promise.all([
versionManager.getCurrentStorageVersion(),
versionManager.getCurrentSchemaVersion(),
storage.count()
]);
schemas.push({
schemaName,
currentStorageVersion: storageVer,
currentSchemaVersion: schemaVer,
entityCount: count
});
totalEntities += count;
}
return { schemas, totalSystemEntities: totalEntities };
}
/**
* Remove a storage instance and clean up its associated managers
*
* @param name - The storage schema name to remove
*/
static removeStorage(name) {
// Clean up version manager
if (this.storageVersionManagers.has(name)) {
this.storageVersionManagers.delete(name);
}
// Clean up export/import manager
if (this.exportImportManagers.has(name)) {
this.exportImportManagers.delete(name);
}
// Clean up storage instance
if (this.storageInstances.has(name)) {
this.storageInstances.delete(name);
}
}
/**
* Clear all static maps and reset the system
* WARNING: This will remove all references to storage instances
*/
static clearAll() {
this.storageVersionManagers.clear();
this.exportImportManagers.clear();
this.storageInstances.clear();
}
/**
* Get the size of static maps for monitoring memory usage
*
* @returns Object with map sizes
*/
static getMapSizes() {
const sizes = {
storageVersionManagers: this.storageVersionManagers.size,
exportImportManagers: this.exportImportManagers.size,
storageInstances: this.storageInstances.size,
total: 0
};
sizes.total = sizes.storageVersionManagers +
sizes.exportImportManagers +
sizes.storageInstances;
return sizes;
}
/**
* Check if a storage instance exists in memory
*
* @param name - The storage schema name
* @returns True if the storage exists in memory
*/
static hasStorageInMemory(name) {
return this.storageInstances.has(name);
}
/**
* Get Redis OM version validation status
*
* @returns Version validation status and information
*
* @example
* ```typescript
* const versionStatus = await StorageSystem.getVersionStatus();
* console.log('Current version:', versionStatus.currentVersion);
* console.log('Is valid:', versionStatus.isValid);
* ```
*/
static async getVersionStatus() {
const versionValidationEnabled = this.config.versionValidation?.enabled !== false;
if (!versionValidationEnabled) {
return {
enabled: false,
currentVersion: 'N/A',
requiredVersion: 'N/A',
supportedRange: 'N/A',
isValid: true,
storedVersion: null,
features: []
};
}
// Get or create validator instance
if (!this.versionValidator) {
const strictMode = this.config.versionValidation?.strictMode !== false;
this.versionValidator = (0, redis_om_version_validator_1.getRedisOMVersionValidator)(strictMode);
// Initialize with a connection if available
try {
const connection = connection_1.RedisConnection.getInstance();
const client = await connection.connect();
await this.versionValidator.initialize(client);
}
catch (error) {
// Connection might not be available yet
console.warn('Could not initialize version validator:', error);
}
}
const status = await this.versionValidator.getCompatibilityReport();
// Get validation details if initialized
let validation = { valid: true, errors: [], warnings: [] };
try {
validation = await this.versionValidator.validateVersion();
}
catch (error) {
// Validation might fail if not initialized
}
return {
enabled: true,
currentVersion: status.currentVersion,
requiredVersion: status.requiredVersion,
supportedRange: status.supportedRange,
isValid: status.isValid,
storedVersion: status.storedVersion,
features: status.features,
errors: validation.errors.length > 0 ? validation.errors : undefined,
warnings: validation.warnings.length > 0 ? validation.warnings : undefined
};
}
/**
* Configure version validation settings
*
* @param settings - Version validation settings
*
* @example
* ```typescript
* // Disable version validation for development
* StorageSystem.configureVersionValidation({ enabled: false });
*
* // Enable strict mode
* StorageSystem.configureVersionValidation({ enabled: true, strictMode: true });
* ```
*/
static configureVersionValidation(settings) {
if (!this.config.versionValidation) {
this.config.versionValidation = {};
}
if (settings.enabled !== undefined) {
this.config.versionValidation.enabled = settings.enabled;
}
if (settings.strictMode !== undefined) {
this.config.versionValidation.strictMode = settings.strictMode;
}
// Reset validator if settings changed
this.versionValidator = null;
}
}
exports.StorageSystem = StorageSystem;
//# sourceMappingURL=storage-system.js.map