claude-flow-novice
Version:
Claude Flow Novice - Advanced orchestration platform for multi-agent AI workflows with CFN Loop architecture Includes CodeSearch (hybrid SQLite + pgvector), mem0/memgraph specialists, and all CFN skills.
451 lines (450 loc) • 16 kB
JavaScript
/**
* Distributed Lock Manager - Enhanced v2
*
* Provides distributed locking mechanism for cross-database transactions using Redis.
* Prevents concurrent modifications and ensures data consistency.
*
* Enhanced Features (Phase 2, Task P2-2.2):
* - Mandatory TTL enforcement with automatic expiration
* - Lock renewal mechanism for long-running operations
* - Auto-renewal with configurable interval
* - Lock health monitoring and statistics
* - Deadlock detection support
* - Stale lock cleanup
* - Backward compatible with existing usage
*
* Part of Task 3.1 (enhanced in Task P2-2.2)
*/ import { randomUUID } from 'crypto';
import { createLogger } from './logging.js';
import { generateCorrelationId } from './correlation.js';
const logger = createLogger('distributed-lock');
/**
* Enhanced Redis-backed distributed lock manager
*/ export class DistributedLockManager {
redisClient;
activeLocks = new Map();
renewalTimers = new Map();
statistics = {
totalAcquisitions: 0,
totalReleases: 0,
currentlyHeld: 0,
totalRenewals: 0,
averageDuration: 0,
failedAcquisitions: 0
};
lockDurations = [];
constructor(redisClient){
this.redisClient = redisClient;
logger.info('Enhanced distributed lock manager initialized (v2)');
}
/**
* Acquire lock on a resource with TTL enforcement and optional auto-renewal
*/ async acquireLock(options) {
// Validate TTL (REQUIRED in v2)
if (options.ttl === undefined || options.ttl === null) {
// Backward compatibility: use default 60s TTL
options.ttl = 60000;
}
if (options.ttl <= 0) {
throw new Error('TTL must be positive');
}
const opts = {
timeout: options.timeout ?? 10000,
transactionId: options.transactionId,
retryInterval: options.retryInterval ?? 100,
correlationId: options.correlationId ?? generateCorrelationId()
};
const lockKey = this.buildLockKey(options.key);
const lockId = randomUUID();
const startTime = Date.now();
logger.debug('Attempting lock acquisition', {
lockKey,
lockId,
ttl: options.ttl,
renewInterval: options.renewInterval,
transactionId: opts.transactionId,
timeout: opts.timeout,
correlationId: opts.correlationId
});
// Try to acquire lock with timeout
while(Date.now() - startTime < opts.timeout){
const acquired = await this.tryAcquire(lockKey, lockId, options, opts);
if (acquired) {
const lock = {
id: lockId,
key: options.key,
acquiredAt: new Date(),
transactionId: opts.transactionId,
ttl: options.ttl,
renewInterval: options.renewInterval,
correlationId: opts.correlationId
};
this.activeLocks.set(lockId, lock);
this.statistics.totalAcquisitions++;
this.statistics.currentlyHeld++;
// Start auto-renewal if configured
if (options.renewInterval) {
this.startAutoRenewal(lock);
}
logger.info('Lock acquired successfully', {
lockId,
lockKey,
ttl: options.ttl,
renewInterval: options.renewInterval,
transactionId: opts.transactionId,
duration: Date.now() - startTime
});
return lock;
}
// Wait before retry
await this.sleep(opts.retryInterval);
}
// Timeout reached
const currentHolder = await this.getLockInfo(options.key);
this.statistics.failedAcquisitions++;
logger.warn('Lock acquisition timeout', {
lockKey,
lockId,
timeout: opts.timeout,
currentHolder,
correlationId: opts.correlationId
});
throw new LockAcquisitionError(`Failed to acquire lock on ${lockKey} within ${opts.timeout}ms`, lockKey, currentHolder);
}
/**
* Try to acquire lock atomically (single attempt)
*/ async tryAcquire(lockKey, lockId, options, opts) {
try {
const now = new Date();
const metadata = {
lockId,
transactionId: opts.transactionId,
processId: process.pid,
acquiredAt: now.toISOString(),
expiresAt: new Date(now.getTime() + options.ttl).toISOString(),
correlationId: opts.correlationId
};
// Use Redis SET NX (set if not exists) with TTL (PX = milliseconds)
// This is atomic and prevents race conditions
const result = await this.redisClient.set(lockKey, JSON.stringify(metadata), 'PX', options.ttl, 'NX' // only set if not exists
);
return result === 'OK';
} catch (err) {
logger.error('Error during lock acquisition attempt', err, {
lockKey,
lockId
});
return false;
}
}
/**
* Renew lock TTL (extends expiration time)
*/ async renewLock(lockId, ttl) {
const lock = this.activeLocks.get(lockId);
if (!lock) {
throw new LockOwnershipError(`Cannot renew lock ${lockId}: lock not found in active locks`, lockId, 'unknown');
}
const lockKey = this.buildLockKey(lock.key);
try {
// Get current lock metadata
const metadata = await this.getLockInfo(lock.key);
if (!metadata) {
throw new Error('Lock has expired or been released');
}
if (metadata.lockId !== lockId) {
throw new LockOwnershipError(`Cannot renew lock ${lockKey}: ownership mismatch`, lockId, metadata.lockId);
}
// Update expiration time
const now = new Date();
metadata.expiresAt = new Date(now.getTime() + ttl).toISOString();
metadata.lastRenewedAt = now.toISOString();
// Update Redis with new TTL
await this.redisClient.set(lockKey, JSON.stringify(metadata), 'PX', ttl);
// Update local lock object
lock.ttl = ttl;
this.statistics.totalRenewals++;
logger.debug('Lock renewed successfully', {
lockId,
lockKey,
newTtl: ttl
});
} catch (err) {
logger.error('Error renewing lock', err, {
lockId,
lockKey
});
throw err;
}
}
/**
* Start auto-renewal for a lock
*/ startAutoRenewal(lock) {
if (!lock.renewInterval) {
return;
}
const timer = setInterval(async ()=>{
try {
await this.renewLock(lock.id, lock.ttl);
logger.debug('Auto-renewal completed', {
lockId: lock.id,
key: lock.key
});
} catch (err) {
logger.error('Auto-renewal failed', err, {
lockId: lock.id,
key: lock.key
});
// Stop renewal on failure
this.stopAutoRenewal(lock.id);
}
}, lock.renewInterval);
this.renewalTimers.set(lock.id, timer);
logger.debug('Auto-renewal started', {
lockId: lock.id,
key: lock.key,
interval: lock.renewInterval
});
}
/**
* Stop auto-renewal for a lock
*/ stopAutoRenewal(lockId) {
const timer = this.renewalTimers.get(lockId);
if (timer) {
clearInterval(timer);
this.renewalTimers.delete(lockId);
logger.debug('Auto-renewal stopped', {
lockId
});
}
}
/**
* Release a lock manually
*/ async releaseLock(lockId) {
const lock = this.activeLocks.get(lockId);
if (!lock) {
logger.warn('Lock not found in active locks', {
lockId
});
return;
}
const lockKey = this.buildLockKey(lock.key);
logger.debug('Releasing lock', {
lockId,
lockKey,
transactionId: lock.transactionId
});
try {
// Stop auto-renewal if active
this.stopAutoRenewal(lockId);
// Verify we own the lock before releasing
const metadata = await this.getLockInfo(lock.key);
if (!metadata) {
logger.warn('Lock already released or expired', {
lockId,
lockKey
});
this.activeLocks.delete(lockId);
this.statistics.currentlyHeld = Math.max(0, this.statistics.currentlyHeld - 1);
return;
}
if (metadata.lockId !== lockId) {
logger.error('Lock ownership mismatch', new Error('Lock ownership mismatch'), {
expectedLockId: lockId,
actualLockId: metadata.lockId,
lockKey
});
throw new LockOwnershipError(`Cannot release lock ${lockKey}: ownership mismatch`, lockId, metadata.lockId);
}
// Delete lock from Redis
await this.redisClient.del(lockKey);
// Track duration for statistics
const duration = Date.now() - lock.acquiredAt.getTime();
this.lockDurations.push(duration);
// Keep only last 100 durations for average calculation
if (this.lockDurations.length > 100) {
this.lockDurations.shift();
}
// Update statistics
this.statistics.totalReleases++;
this.statistics.currentlyHeld = Math.max(0, this.statistics.currentlyHeld - 1);
this.statistics.averageDuration = this.calculateAverageDuration();
this.activeLocks.delete(lockId);
logger.info('Lock released successfully', {
lockId,
lockKey,
duration,
transactionId: lock.transactionId
});
} catch (err) {
logger.error('Error releasing lock', err, {
lockId,
lockKey
});
throw err;
}
}
/**
* Check if a resource is currently locked
*/ async isLocked(key) {
const lockKey = this.buildLockKey(key);
try {
const exists = await this.redisClient.exists(lockKey);
return exists === 1;
} catch (err) {
logger.error('Error checking lock status', err, {
lockKey
});
return false; // Assume unlocked on error (fail-safe)
}
}
/**
* Get lock metadata for a resource
*/ async getLockInfo(key) {
const lockKey = this.buildLockKey(key);
try {
const data = await this.redisClient.get(lockKey);
if (!data) {
return null;
}
const metadata = JSON.parse(data);
return metadata;
} catch (err) {
logger.error('Error retrieving lock info', err, {
lockKey
});
return null;
}
}
/**
* Force release of a lock (admin operation - use with caution)
*/ async forceRelease(key) {
const lockKey = this.buildLockKey(key);
logger.warn('Force releasing lock', {
lockKey
});
try {
await this.redisClient.del(lockKey);
// Clean up from active locks if present
for (const [id, lock] of Array.from(this.activeLocks.entries())){
if (lock.key === key) {
this.stopAutoRenewal(id);
this.activeLocks.delete(id);
this.statistics.currentlyHeld = Math.max(0, this.statistics.currentlyHeld - 1);
}
}
logger.info('Lock force released', {
lockKey
});
} catch (err) {
logger.error('Error force releasing lock', err, {
lockKey
});
throw err;
}
}
/**
* Get all active locks managed by this instance
*/ getActiveLocks() {
return Array.from(this.activeLocks.values());
}
/**
* Get lock statistics
*/ getStatistics() {
return {
...this.statistics
};
}
/**
* Clean up expired locks from tracking
*/ cleanupExpiredLocks() {
const now = Date.now();
let cleaned = 0;
for (const [id, lock] of Array.from(this.activeLocks.entries())){
const expiresAt = lock.acquiredAt.getTime() + lock.ttl;
if (now > expiresAt) {
this.stopAutoRenewal(id);
this.activeLocks.delete(id);
this.statistics.currentlyHeld = Math.max(0, this.statistics.currentlyHeld - 1);
cleaned++;
}
}
if (cleaned > 0) {
logger.debug('Cleaned up expired locks from tracking', {
count: cleaned
});
}
return cleaned;
}
/**
* Release all active locks (cleanup on shutdown)
*/ async releaseAll() {
logger.info('Releasing all active locks', {
count: this.activeLocks.size
});
const releases = Array.from(this.activeLocks.values()).map((lock)=>this.releaseLock(lock.id).catch((err)=>{
logger.error('Error releasing lock during cleanup', err, {
lockId: lock.id
});
}));
await Promise.all(releases);
logger.info('All locks released');
}
/**
* Build Redis key for lock resource
*/ buildLockKey(key) {
return `lock:${key}`;
}
/**
* Calculate average lock duration
*/ calculateAverageDuration() {
if (this.lockDurations.length === 0) {
return 0;
}
const sum = this.lockDurations.reduce((acc, dur)=>acc + dur, 0);
return Math.round(sum / this.lockDurations.length);
}
/**
* Sleep utility for retry intervals
*/ sleep(ms) {
return new Promise((resolve)=>setTimeout(resolve, ms));
}
}
/**
* Lock acquisition error
*/ export class LockAcquisitionError extends Error {
lockKey;
currentHolder;
constructor(message, lockKey, currentHolder){
super(message), this.lockKey = lockKey, this.currentHolder = currentHolder;
this.name = 'LockAcquisitionError';
}
}
/**
* Lock ownership error
*/ export class LockOwnershipError extends Error {
expectedLockId;
actualLockId;
constructor(message, expectedLockId, actualLockId){
super(message), this.expectedLockId = expectedLockId, this.actualLockId = actualLockId;
this.name = 'LockOwnershipError';
}
}
/**
* Utility: Execute function with distributed lock
*/ export async function withLock(lockManager, key, fn, options) {
const lock = await lockManager.acquireLock({
key,
ttl: options?.ttl ?? 60000,
...options
});
try {
const result = await fn();
await lockManager.releaseLock(lock.id);
return result;
} catch (err) {
await lockManager.releaseLock(lock.id);
throw err;
}
}
export { DistributedLockManager as DistributedLock };
//# sourceMappingURL=distributed-lock.js.map