@yihuangdb/storage-object
Version:
A Node.js storage object layer library using Redis OM
376 lines • 15.7 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SchemaVersionManager = exports.SchemaLockError = void 0;
const redis_key_manager_1 = require("./redis-key-manager");
class SchemaLockError extends Error {
expectedVersion;
actualVersion;
schemaName;
constructor(message, expectedVersion, actualVersion, schemaName) {
super(message);
this.expectedVersion = expectedVersion;
this.actualVersion = actualVersion;
this.schemaName = schemaName;
this.name = 'SchemaLockError';
}
}
exports.SchemaLockError = SchemaLockError;
class SchemaVersionManager {
client;
schemaName;
hashKey;
hllKey;
versionsKey;
keyManager = (0, redis_key_manager_1.getRedisKeyManager)();
// Note: Using WATCH/MULTI/EXEC for optimistic locking instead of persistent lock keys
// This follows Redis best practices and avoids lock key cleanup issues
defaultMaxRetries = 3;
defaultLockTimeout = 5000; // 5 seconds
constructor(schemaName, client) {
this.schemaName = schemaName;
this.client = client;
// Use RedisKeyManager for consistent key generation
this.hashKey = this.keyManager.getSchemaMetadataKey(schemaName);
this.hllKey = this.keyManager.getSchemaHllKey(schemaName);
this.versionsKey = this.keyManager.getSchemaVersionsKey(schemaName);
// Lock is implemented via WATCH on hashKey, not a separate lock key
}
/**
* Store schema metadata in HashMap with locking
*/
async saveMetadata(metadata, options) {
const maxRetries = options?.maxRetries ?? this.defaultMaxRetries;
const retryOnConflict = options?.retryOnConflict ?? true;
if (!retryOnConflict) {
// Single attempt without retry
return this._saveMetadataWithLock(metadata, options?.expectedVersion);
}
// Retry logic with exponential backoff
let lastError;
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
await this._saveMetadataWithLock(metadata, options?.expectedVersion);
return;
}
catch (error) {
lastError = error;
if (error instanceof SchemaLockError && attempt < maxRetries - 1) {
// Exponential backoff: 50ms, 100ms, 200ms...
const delay = Math.min(50 * Math.pow(2, attempt), 1000);
await new Promise(resolve => setTimeout(resolve, delay));
continue;
}
throw error;
}
}
throw lastError || new Error('Failed to save metadata after retries');
}
/**
* Internal method to save metadata with WATCH/MULTI/EXEC
*/
async _saveMetadataWithLock(metadata, expectedVersion) {
// Watch the hash key for changes
await this.client.watch(this.hashKey);
try {
// Get current metadata
const current = await this.getMetadata();
const currentVersion = current?.version ?? 0;
// Check version if specified
if (expectedVersion !== undefined && currentVersion !== expectedVersion) {
await this.client.unwatch();
throw new SchemaLockError('Schema version mismatch', expectedVersion, currentVersion, this.schemaName);
}
// Prepare fields for update
const fields = {};
Object.entries(metadata).forEach(([key, value]) => {
if (value !== undefined) {
fields[key] = typeof value === 'number' ? value.toString() : value;
}
});
// Update timestamp
fields.updatedAt = new Date().toISOString();
// Execute transaction with validated key
const multi = this.client.multi();
multi.hSet(this.hashKey, fields);
const result = await multi.exec();
if (!result || result.length === 0) {
// Transaction failed due to watched key being modified
throw new SchemaLockError('Schema modified by another process', expectedVersion ?? currentVersion, -1, this.schemaName);
}
}
catch (error) {
await this.client.unwatch();
throw error;
}
}
/**
* Get schema metadata from HashMap
*/
async getMetadata() {
// Use safe operation with key validation
const data = await this.keyManager.safeHGetAll(this.client, this.hashKey);
if (!data || Object.keys(data).length === 0) {
return null;
}
// Convert string values back to proper types
return {
version: parseInt(data.version || '1', 10),
fields: data.fields || '',
indexes: data.indexes || '',
dataStructure: data.dataStructure || 'HASH',
entityCount: data.entityCount || '0',
dataVersion: data.dataVersion || '1',
lastDataChange: data.lastDataChange || '',
libraryVersion: data.libraryVersion || '',
createdAt: data.createdAt || '',
updatedAt: data.updatedAt || '',
lastAccessed: data.lastAccessed || ''
};
}
/**
* Increment version ONLY for structure changes with atomic locking
*/
async incrementVersion(reason, options) {
const maxRetries = options?.maxRetries ?? this.defaultMaxRetries;
for (let attempt = 0; attempt < maxRetries; attempt++) {
// Watch the hash key for concurrent modifications
await this.client.watch(this.hashKey);
try {
// Get current version with validated key
const currentVersionStr = await this.keyManager.safeHGet(this.client, this.hashKey, 'version');
const currentVersion = currentVersionStr ? parseInt(currentVersionStr, 10) : 0;
// Check expected version if provided
if (options?.expectedVersion !== undefined && currentVersion !== options.expectedVersion) {
await this.client.unwatch();
throw new SchemaLockError('Version mismatch during increment', options.expectedVersion, currentVersion, this.schemaName);
}
const newVersion = currentVersion + 1;
const timestamp = Date.now();
const isoDate = new Date().toISOString();
// Execute atomic transaction
const multi = this.client.multi();
// Update version and timestamp atomically with validated key
multi.hSet(this.hashKey, {
version: newVersion.toString(),
updatedAt: isoDate,
lastVersionChange: isoDate,
versionChangeReason: reason
});
// Add to version history
multi.zAdd(this.versionsKey, {
score: timestamp,
value: `v${newVersion}:${reason}`
});
// Keep only last 100 versions
multi.zRemRangeByRank(this.versionsKey, 0, -101);
const result = await multi.exec();
if (!result || result.length === 0) {
// Transaction failed - key was modified
if (attempt < maxRetries - 1) {
// Exponential backoff
const delay = Math.min(50 * Math.pow(2, attempt), 1000);
await new Promise(resolve => setTimeout(resolve, delay));
continue;
}
throw new SchemaLockError('Failed to increment version - concurrent modification', currentVersion, -1, this.schemaName);
}
return newVersion;
}
catch (error) {
await this.client.unwatch();
if (error instanceof SchemaLockError) {
throw error;
}
if (attempt < maxRetries - 1) {
const delay = Math.min(50 * Math.pow(2, attempt), 1000);
await new Promise(resolve => setTimeout(resolve, delay));
continue;
}
throw error;
}
}
throw new Error(`Failed to increment version after ${maxRetries} attempts`);
}
/**
* Track entity creation with HyperLogLog (atomic operation)
*/
async trackEntity(entityId) {
// HyperLogLog operations are already atomic, but we'll update metadata atomically
const maxRetries = 3;
for (let attempt = 0; attempt < maxRetries; attempt++) {
await this.client.watch(this.hashKey);
try {
// Add to HyperLogLog (this is atomic by itself)
await this.client.pfAdd(this.hllKey, entityId);
// Get updated count
const count = await this.client.pfCount(this.hllKey);
// Update metadata atomically
const multi = this.client.multi();
multi.hSet(this.hashKey, {
entityCount: `~${count}`,
lastDataChange: new Date().toISOString()
});
const result = await multi.exec();
if (!result || result.length === 0) {
// Retry if transaction failed
if (attempt < maxRetries - 1) {
const delay = Math.min(50 * Math.pow(2, attempt), 500);
await new Promise(resolve => setTimeout(resolve, delay));
continue;
}
}
return;
}
catch (error) {
await this.client.unwatch();
if (attempt === maxRetries - 1) {
throw error;
}
}
}
}
/**
* Get entity count from HyperLogLog
*/
async getEntityCount() {
const count = await this.client.pfCount(this.hllKey);
return count;
}
/**
* Add version to history sorted set (now handled atomically in incrementVersion)
*/
async recordVersionHistory(version, timestamp, reason) {
// This is now handled atomically in incrementVersion
// Kept for backward compatibility but should not be called directly
const value = reason ? `v${version}:${reason}` : `v${version}`;
await this.client.zAdd(this.versionsKey, {
score: timestamp,
value
});
// Keep only last 100 versions to prevent unbounded growth
const count = await this.client.zCard(this.versionsKey);
if (count > 100) {
await this.client.zRemRangeByRank(this.versionsKey, 0, count - 101);
}
}
/**
* Get version history from sorted set
*/
async getVersionHistory() {
// Get all versions with scores (timestamps)
const history = await this.client.zRangeWithScores(this.versionsKey, 0, -1);
return history.map(entry => {
const [versionPart, ...reasonParts] = entry.value.split(':');
return {
version: parseInt(versionPart.replace('v', ''), 10),
timestamp: entry.score,
reason: reasonParts.length > 0 ? reasonParts.join(':') : undefined
};
});
}
/**
* Check if structure changed (for version increment)
*/
isStructuralChange(oldFields, newFields) {
// Compare serialized field definitions
// Only structural changes trigger version increment
// Entity count changes do NOT trigger version changes
if (!oldFields || !newFields) {
return true; // Initial setup or missing data
}
try {
const oldFieldsObj = JSON.parse(oldFields);
const newFieldsObj = JSON.parse(newFields);
// Check for field additions/removals
const oldKeys = Object.keys(oldFieldsObj).sort();
const newKeys = Object.keys(newFieldsObj).sort();
if (oldKeys.length !== newKeys.length) {
return true;
}
// Check if keys are the same
for (let i = 0; i < oldKeys.length; i++) {
if (oldKeys[i] !== newKeys[i]) {
return true;
}
}
// Check if field types changed
for (const key of oldKeys) {
const oldField = oldFieldsObj[key];
const newField = newFieldsObj[key];
// Compare type and indexed status
if (oldField.type !== newField.type ||
oldField.indexed !== newField.indexed) {
return true;
}
}
return false; // No structural changes
}
catch (error) {
// If parsing fails, consider it a structural change
return true;
}
}
/**
* Initialize schema metadata
*/
async initialize(fields, indexes, dataStructure) {
const existing = await this.getMetadata();
const fieldsStr = JSON.stringify(fields);
const indexesStr = JSON.stringify(indexes);
if (existing) {
// If fields are not set yet, this is still initial setup - don't increment version
if (!existing.fields) {
// Initial setup - just update the fields
await this.saveMetadata({
fields: fieldsStr,
indexes: indexesStr,
dataStructure,
lastAccessed: new Date().toISOString()
});
return;
}
// Check for structural changes only if fields were previously set
const hasStructuralChange = this.isStructuralChange(existing.fields, fieldsStr) ||
existing.indexes !== indexesStr ||
existing.dataStructure !== dataStructure;
if (hasStructuralChange) {
// Structural change - increment version
await this.incrementVersion('Schema structure changed');
}
// Update metadata
await this.saveMetadata({
fields: fieldsStr,
indexes: indexesStr,
dataStructure,
lastAccessed: new Date().toISOString()
});
}
else {
// First time initialization
await this.saveMetadata({
version: 1,
fields: fieldsStr,
indexes: indexesStr,
dataStructure,
entityCount: '0',
dataVersion: '1',
lastDataChange: new Date().toISOString(),
libraryVersion: '0.1.0',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
lastAccessed: new Date().toISOString()
});
// Record initial version
await this.recordVersionHistory(1, Date.now());
}
}
/**
* Clean up all schema-related keys
*/
async cleanup() {
// Delete hash, HLL, and version history
await this.client.del([this.hashKey, this.hllKey, this.versionsKey]);
}
}
exports.SchemaVersionManager = SchemaVersionManager;
//# sourceMappingURL=schema-versioning.js.map