claude-flow-novice
Version:
Claude Flow Novice - Advanced orchestration platform for multi-agent AI workflows with CFN Loop architecture Includes Local RuVector Accelerator and all CFN skills for complete functionality.
292 lines (291 loc) • 10.6 kB
JavaScript
/**
* Deadlock Resolver
*
* Detects and resolves deadlocks in cross-database transactions using:
* - Timeout-based deadlock detection
* - Automatic resolution (abort younger transaction)
* - Exponential backoff retry logic
* - Deadlock logging and metrics
*
* Part of Task 3.1: Cross-Database Transaction Framework
*/ import { LockAcquisitionError } from './distributed-lock.js';
import { withRetry } from './retry.js';
import { createLogger } from './logging.js';
import { generateCorrelationId } from './correlation.js';
const logger = createLogger('deadlock-resolver');
/**
* Deadlock Resolver - handles detection and automatic resolution
*/ export class DeadlockResolver {
txManager;
lockManager;
stats = {
totalDetected: 0,
totalResolved: 0,
totalAborted: 0,
avgResolutionTimeMs: 0
};
constructor(txManager, lockManager){
this.txManager = txManager;
this.lockManager = lockManager;
logger.info('Deadlock resolver initialized');
}
/**
* Detect potential deadlock for a transaction
*
* Uses timeout-based detection: if transaction is waiting for lock beyond threshold,
* consider it a deadlock candidate.
*/ async detectDeadlock(transaction, waitTimeMs = 5000) {
const duration = transaction.getDuration();
if (duration < waitTimeMs) {
return false; // Not waiting long enough
}
logger.debug('Potential deadlock detected (timeout)', {
transactionId: transaction.id,
duration,
threshold: waitTimeMs
});
this.stats.totalDetected++;
this.stats.lastDeadlock = new Date();
return true;
}
/**
* Resolve deadlock by aborting younger transaction
*
* Strategy: In a deadlock involving multiple transactions, abort the youngest one
* (most recently started) to minimize wasted work.
*/ async resolve(transactions) {
if (transactions.length === 0) {
throw new Error('Cannot resolve deadlock with no transactions');
}
if (transactions.length === 1) {
// Single transaction deadlock (waiting on itself) - just abort it
await this.abortTransaction(transactions[0]);
return {
resolved: true,
abortedTransaction: transactions[0],
survivingTransactions: [],
method: 'youngest-abort',
resolvedAt: new Date()
};
}
logger.info('Resolving deadlock', {
transactionCount: transactions.length,
transactionIds: transactions.map((t)=>t.id)
});
const startTime = Date.now();
// Sort by start time (oldest first)
const sorted = transactions.sort((a, b)=>a.startedAt.getTime() - b.startedAt.getTime());
// Abort youngest transaction (last in sorted array)
const youngest = sorted[sorted.length - 1];
const survivors = sorted.slice(0, -1);
await this.abortTransaction(youngest);
const resolutionTime = Date.now() - startTime;
// Update statistics
this.stats.totalResolved++;
this.stats.totalAborted++;
this.stats.avgResolutionTimeMs = (this.stats.avgResolutionTimeMs * (this.stats.totalResolved - 1) + resolutionTime) / this.stats.totalResolved;
logger.info('Deadlock resolved', {
abortedTransactionId: youngest.id,
survivingTransactions: survivors.map((t)=>t.id),
resolutionTimeMs: resolutionTime
});
return {
resolved: true,
abortedTransaction: youngest,
survivingTransactions: survivors,
method: 'youngest-abort',
resolvedAt: new Date()
};
}
/**
* Execute operation with automatic deadlock retry
*
* If operation fails due to deadlock, automatically retry with exponential backoff.
*/ async executeWithRetry(operation, options = {}) {
const opts = {
maxAttempts: options.maxAttempts ?? 3,
baseDelayMs: options.baseDelayMs ?? 100,
maxDelayMs: options.maxDelayMs ?? 5000,
backoffFactor: options.backoffFactor ?? 2,
exponential: options.exponential ?? true,
jitter: options.jitter ?? true
};
const correlationId = generateCorrelationId();
logger.debug('Executing operation with deadlock retry', {
maxAttempts: opts.maxAttempts,
baseDelayMs: opts.baseDelayMs,
correlationId
});
return await withRetry(operation, {
maxAttempts: opts.maxAttempts,
baseDelayMs: opts.baseDelayMs,
maxDelayMs: opts.maxDelayMs,
exponential: opts.exponential,
jitter: opts.jitter,
shouldRetry: (error)=>{
// Retry on deadlock or lock acquisition errors
return error instanceof DeadlockError || error instanceof LockAcquisitionError || error.message.includes('deadlock') || error.message.includes('lock');
},
onRetry: (attempt, error, delayMs)=>{
logger.warn('Retrying after deadlock', {
attempt,
maxAttempts: opts.maxAttempts,
error: error.message,
delayMs,
correlationId
});
}
});
}
/**
* Execute transaction with automatic deadlock handling
*
* Wraps transaction execution with deadlock detection and automatic retry.
*/ async executeTransaction(databases, operations, options = {}) {
return await this.executeWithRetry(async ()=>{
const tx = await this.txManager.begin(databases);
try {
const results = [];
for (const operation of operations){
const result = await operation(tx);
results.push(result);
}
await tx.commit();
return results;
} catch (err) {
await tx.rollback();
// Check if this is a deadlock
if (err instanceof LockAcquisitionError) {
throw new DeadlockError(`Deadlock detected: ${err.message}`, [
tx
], []);
}
throw err;
}
}, options);
}
/**
* Get deadlock statistics
*/ getStats() {
return {
...this.stats
};
}
/**
* Reset statistics
*/ resetStats() {
this.stats = {
totalDetected: 0,
totalResolved: 0,
totalAborted: 0,
avgResolutionTimeMs: 0
};
logger.info('Deadlock statistics reset');
}
/**
* Abort a transaction (rollback and cleanup)
*/ async abortTransaction(transaction) {
logger.info('Aborting transaction', {
transactionId: transaction.id,
age: transaction.getDuration()
});
try {
await transaction.rollback();
logger.debug('Transaction aborted successfully', {
transactionId: transaction.id
});
} catch (err) {
logger.error('Error aborting transaction', err, {
transactionId: transaction.id
});
throw err;
}
}
/**
* Monitor active transactions for potential deadlocks
*
* Should be called periodically (e.g., every second) to detect long-running
* transactions that may be in deadlock.
*/ async monitorDeadlocks(thresholdMs = 10000) {
const activeTransactions = this.txManager.getActiveTransactions();
const deadlocks = [];
for (const tx of activeTransactions){
const isDeadlocked = await this.detectDeadlock(tx, thresholdMs);
if (isDeadlocked) {
deadlocks.push({
detected: true,
transactions: [
tx
],
resources: [],
method: 'timeout',
detectedAt: new Date()
});
}
}
if (deadlocks.length > 0) {
logger.warn('Deadlocks detected during monitoring', {
count: deadlocks.length,
transactionIds: deadlocks.flatMap((d)=>d.transactions.map((t)=>t.id))
});
}
return deadlocks;
}
/**
* Auto-resolve detected deadlocks
*
* Automatically resolve deadlocks found during monitoring.
*/ async autoResolveDeadlocks(thresholdMs = 10000) {
const deadlocks = await this.monitorDeadlocks(thresholdMs);
const resolutions = [];
for (const deadlock of deadlocks){
try {
const resolution = await this.resolve(deadlock.transactions);
resolutions.push(resolution);
} catch (err) {
logger.error('Failed to auto-resolve deadlock', err, {
transactionIds: deadlock.transactions.map((t)=>t.id)
});
}
}
return resolutions;
}
}
/**
* Deadlock error
*/ export class DeadlockError extends Error {
transactions;
resources;
constructor(message, transactions, resources){
super(message), this.transactions = transactions, this.resources = resources;
this.name = 'DeadlockError';
}
}
/**
* Utility: Create deadlock resolver with default configuration
*/ export function createDeadlockResolver(txManager, lockManager) {
return new DeadlockResolver(txManager, lockManager);
}
/**
* Utility: Start automatic deadlock monitoring
*
* Returns cleanup function to stop monitoring.
*/ export function startDeadlockMonitoring(resolver, intervalMs = 1000, thresholdMs = 10000) {
logger.info('Starting automatic deadlock monitoring', {
intervalMs,
thresholdMs
});
const intervalHandle = setInterval(async ()=>{
try {
await resolver.autoResolveDeadlocks(thresholdMs);
} catch (err) {
logger.error('Error during automatic deadlock resolution', err);
}
}, intervalMs);
// Return cleanup function
return ()=>{
clearInterval(intervalHandle);
logger.info('Stopped automatic deadlock monitoring');
};
}
//# sourceMappingURL=deadlock-resolver.js.map