@eleven-am/transcoder
Version:
High-performance HLS transcoding library with hardware acceleration, intelligent client management, and distributed processing support for Node.js
303 lines • 12.9 kB
JavaScript
"use strict";
/*
* @eleven-am/transcoder
* Copyright (C) 2025 Roy OSSAI
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.WorkStealingProcessor = void 0;
const events_1 = require("events");
const fp_1 = require("@eleven-am/fp");
const distributedSegmentProcessor_1 = require("./distributedSegmentProcessor");
/**
* Extends distributed processing with work stealing capability
* Monitors for stuck segments and steals them from unresponsive workers
*/
class WorkStealingProcessor extends events_1.EventEmitter {
constructor(redis, config = {}) {
super();
this.stealCheckTimer = null;
this.isDisposed = false;
this.MAX_STEAL_ATTEMPTS = 500;
this.activeStealAttempts = new Set();
// Circuit breaker properties
this.consecutiveFailures = 0;
this.maxFailures = 3;
this.failureBackoff = 30000; // 30 seconds
// Pure helper functions for work stealing
this.limitSegmentsToSteal = (segments) => segments.slice(0, this.config.maxStealsPerCheck);
this.emitDetectionEvent = (stuckCount, stealingCount) => {
this.emit('work:stealing:detected', {
workerId: this.config.workerId,
stuckCount,
stealingCount,
});
};
this.countResults = (results) => ({
successful: results.filter((r) => r.status === 'fulfilled').length,
failed: results.filter((r) => r.status === 'rejected').length,
});
this.emitCompletionEvent = (successful, failed) => {
if (successful > 0 || failed > 0) {
this.emit('work:stealing:complete', {
workerId: this.config.workerId,
successful,
failed,
});
}
};
this.handleFailure = (error) => {
this.consecutiveFailures++;
console.error(`Work stealing check failed (${this.consecutiveFailures}/${this.maxFailures}): ${error}`);
this.emit('work:stealing:error', {
workerId: this.config.workerId,
error,
});
};
this.shouldTripCircuitBreaker = () => this.consecutiveFailures >= this.maxFailures;
this.tripCircuitBreaker = () => {
console.error(`Circuit breaker tripped for work stealing. Pausing for ${this.failureBackoff}ms.`);
if (this.stealCheckTimer) {
clearInterval(this.stealCheckTimer);
this.stealCheckTimer = null;
}
// Schedule restart after backoff period
setTimeout(() => {
if (!this.isDisposed) {
this.consecutiveFailures = 0;
this.startWorkStealingMonitor();
}
}, this.failureBackoff);
};
this.resetFailureCount = () => {
this.consecutiveFailures = 0;
};
this.findStuckSegmentsTask = () => fp_1.TaskEither.tryCatch(() => this.findStuckSegments(), 'Failed to find stuck segments');
this.attemptStealSegmentTask = (segment) => fp_1.TaskEither.tryCatch(() => this.attemptStealSegment(segment), 'Failed to steal segment');
this.attemptStealSegments = (segments) => fp_1.TaskEither.tryCatch(() => Promise.allSettled(segments.map((segment) => this.attemptStealSegment(segment))), 'Failed to steal segments');
// Pure helper functions for finding stuck segments
this.calculateClaimAge = (segment, now) => now - segment.claimTime;
this.isSegmentStuck = (segment, now) => this.calculateClaimAge(segment, now) > this.config.stealThreshold;
this.isNotBeingStolen = (segment) => !this.activeStealAttempts.has(this.getSegmentKey(segment));
this.parseSegmentData = (segment) => {
const claimManager = this.baseProcessor.getClaimManager();
const parsedKey = claimManager.parseSegmentKey(segment.segmentKey);
if (parsedKey) {
return {
fileId: parsedKey.fileId,
sourceFilePath: '', // Will be filled by processor
streamType: parsedKey.streamType,
quality: parsedKey.quality,
streamIndex: parsedKey.streamIndex,
segmentIndex: parsedKey.segmentIndex,
segmentStart: 0, // Will be filled by processor
segmentDuration: 0, // Will be filled by processor
totalSegments: 0, // Will be filled by processor
ffmpegOptions: {
inputOptions: [],
outputOptions: [],
},
outputPath: segment.outputPath,
// Store the original segment for stealing
originalSegment: segment,
};
}
return null;
};
this.filterStuckSegments = (segments, now) => {
const stuckSegments = [];
for (const segment of segments) {
if (this.isSegmentStuck(segment, now) && this.isNotBeingStolen(segment)) {
const parsedData = this.parseSegmentData(segment);
if (parsedData) {
stuckSegments.push(parsedData);
}
}
}
return stuckSegments;
};
this.getClaimedSegmentsTask = () => fp_1.TaskEither.tryCatch(() => this.baseProcessor.getClaimedSegments(), 'Failed to get claimed segments');
// Pure helper functions for segment stealing
this.markSegmentAsBeingStolen = (segmentKey) => {
// Implement bounded Set with LRU eviction
if (this.activeStealAttempts.size >= this.MAX_STEAL_ATTEMPTS) {
this.evictOldestStealAttempt();
}
this.activeStealAttempts.add(segmentKey);
};
this.unmarkSegmentAsBeingStolen = (segmentKey) => {
this.activeStealAttempts.delete(segmentKey);
};
this.getOriginalSegmentKey = (segment) => segment.originalSegment?.segmentKey || this.getSegmentKey(segment);
this.emitWorkStolenEvent = (segment) => {
this.emit('work:stolen', {
workerId: this.config.workerId,
segmentNumber: segment.segmentIndex,
originalWorker: segment.originalSegment?.workerId || 'unknown',
});
};
this.forceClaimSegmentTask = (segmentKey, outputPath) => fp_1.TaskEither.tryCatch(() => this.baseProcessor.forceClaimSegment(segmentKey, this.config.workerId, outputPath), 'Failed to force claim segment');
this.processSegmentTask = (segment) => fp_1.TaskEither.tryCatch(() => this.processSegment(segment), 'Failed to process stolen segment');
// Memory management helper methods
this.evictOldestStealAttempt = () => {
const firstKey = this.activeStealAttempts.values().next().value;
if (firstKey) {
this.activeStealAttempts.delete(firstKey);
}
};
this.clearAllStealAttempts = () => {
this.activeStealAttempts.clear();
};
this.config = {
workerId: config.workerId || `worker-${process.pid}`,
stealThreshold: config.stealThreshold || 60000, // 1 minute
stealCheckInterval: config.stealCheckInterval || 30000, // 30 seconds
maxStealsPerCheck: config.maxStealsPerCheck || 3,
};
this.baseProcessor = new distributedSegmentProcessor_1.DistributedSegmentProcessor(redis, {
workerId: this.config.workerId,
});
this.startWorkStealingMonitor();
}
/**
* Process a segment with work stealing support
*/
async processSegment(data) {
return this.baseProcessor.processSegment(data);
}
/**
* Check if healthy with work stealing capability
*/
async isHealthy() {
return this.baseProcessor.isHealthy();
}
/**
* Get processing mode
*/
getMode() {
return 'distributed-work-stealing';
}
/**
* Start monitoring for stuck segments
*/
startWorkStealingMonitor() {
if (this.stealCheckTimer) {
return;
}
this.stealCheckTimer = setInterval(async () => {
if (!this.isDisposed) {
await this.checkAndStealWork();
}
}, this.config.stealCheckInterval);
// Don't block process termination
if (this.stealCheckTimer.unref) {
this.stealCheckTimer.unref();
}
}
/**
* Check for stuck segments and attempt to steal them
*/
async checkAndStealWork() {
await this.findStuckSegmentsTask()
.filter((segments) => segments.length > 0, () => (0, fp_1.createInternalError)('No stuck segments found'))
.map(this.limitSegmentsToSteal)
.ioSync((segmentsToSteal) => {
const stuckCount = segmentsToSteal.length;
const stealingCount = Math.min(stuckCount, this.config.maxStealsPerCheck);
this.emitDetectionEvent(stuckCount, stealingCount);
})
.chain(this.attemptStealSegments)
.map(this.countResults)
.ioSync(({ successful, failed }) => {
this.emitCompletionEvent(successful, failed);
this.resetFailureCount();
})
.orElse((error) => {
this.handleFailure(error.error || error);
if (this.shouldTripCircuitBreaker()) {
this.tripCircuitBreaker();
}
return fp_1.TaskEither.of({ successful: 0, failed: 0 });
})
.toPromise();
}
/**
* Find segments that appear to be stuck
*/
async findStuckSegments() {
return this.getClaimedSegmentsTask()
.map((segments) => this.filterStuckSegments(segments, Date.now()))
.orElse(() => {
console.error('Failed to find stuck segments');
return fp_1.TaskEither.of([]);
})
.toPromise();
}
/**
* Attempt to steal a specific segment
*/
async attemptStealSegment(segment) {
const segmentKey = this.getSegmentKey(segment);
const originalSegmentKey = this.getOriginalSegmentKey(segment);
// Mark as being stolen
this.markSegmentAsBeingStolen(segmentKey);
await this.forceClaimSegmentTask(originalSegmentKey, segment.outputPath)
.filter((claimed) => claimed, () => (0, fp_1.createInternalError)('Failed to claim segment'))
.ioSync(() => this.emitWorkStolenEvent(segment))
.chain(() => this.processSegmentTask(segment))
.ioSync(() => this.unmarkSegmentAsBeingStolen(segmentKey))
.orElse((error) => {
console.error(`Failed to steal segment ${segment.segmentIndex}: ${error}`);
this.unmarkSegmentAsBeingStolen(segmentKey);
return fp_1.TaskEither.error(error);
})
.toPromise();
}
/**
* Get unique key for a segment
*/
getSegmentKey(segment) {
if ('segmentKey' in segment) {
return segment.segmentKey;
}
return `${segment.fileId}:${segment.streamType}:${segment.quality}:${segment.streamIndex}:${segment.segmentIndex}`;
}
/**
* Dispose and clean up
*/
async dispose() {
this.isDisposed = true;
if (this.stealCheckTimer) {
clearInterval(this.stealCheckTimer);
this.stealCheckTimer = null;
}
this.clearAllStealAttempts();
await this.baseProcessor.dispose();
}
/**
* Get work stealing metrics
*/
getMetrics() {
return {
mode: this.getMode(),
workerId: this.config.workerId,
activeStealAttempts: this.activeStealAttempts.size,
stealThreshold: this.config.stealThreshold,
checkInterval: this.config.stealCheckInterval,
};
}
}
exports.WorkStealingProcessor = WorkStealingProcessor;
//# sourceMappingURL=workStealingProcessor.js.map