@eleven-am/transcoder
Version:
High-performance HLS transcoding library with hardware acceleration, intelligent client management, and distributed processing support for Node.js
386 lines (380 loc) • 16.6 kB
JavaScript
;
/*
* @eleven-am/transcoder
* Copyright (C) 2025 Roy OSSAI
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.RedisSegmentClaimManager = void 0;
const fp_1 = require("@eleven-am/fp");
/**
* Manages distributed segment claims using Redis
* Ensures only one worker processes each segment at a time
*/
class RedisSegmentClaimManager {
constructor(redis, workerId, defaultTTL = 60000, // 60 seconds
completedSegmentTTL) {
this.redis = redis;
this.workerId = workerId;
this.defaultTTL = defaultTTL;
this.lockPrefix = 'transcoder:segment:lock:';
this.statusPrefix = 'transcoder:segment:status:';
this.completedPrefix = 'transcoder:segment:completed:';
// Pure helper functions for subscriptions
this.createCompletionChannel = (segmentKey) => `transcoder:segment:complete:${segmentKey}`;
this.handleCompletionMessage = (message, callback) => {
if (message === 'completed') {
callback();
}
};
this.createUnsubscribeFunction = (subscriber, channel) => async () => {
await fp_1.TaskEither.of(subscriber.isOpen)
.filter((isOpen) => isOpen, () => (0, fp_1.createInternalError)('Subscriber not open'))
.chain(() => fp_1.TaskEither.tryCatch(() => subscriber.unsubscribe(channel), 'Failed to unsubscribe'))
.chain(() => fp_1.TaskEither.tryCatch(() => subscriber.disconnect(), 'Failed to disconnect'))
.orElse((error) => {
console.error('Error during Redis subscriber cleanup:', error);
if (subscriber.isOpen) {
return fp_1.TaskEither.tryCatch(() => subscriber.disconnect(), 'Failed to disconnect on cleanup');
}
return fp_1.TaskEither.of(undefined);
})
.toPromise();
};
this.connectSubscriber = (subscriber) => fp_1.TaskEither.tryCatch(() => subscriber.connect().then(() => subscriber), 'Failed to connect subscriber');
this.subscribeToChannel = (subscriber, channel, callback) => fp_1.TaskEither.tryCatch(() => subscriber.subscribe(channel, (message) => this.handleCompletionMessage(message, callback)), 'Failed to subscribe to channel');
this.disconnectSubscriber = (subscriber) => fp_1.TaskEither.of(subscriber.isOpen)
.filter((isOpen) => isOpen, () => (0, fp_1.createInternalError)('Subscriber not open'))
.chain(() => fp_1.TaskEither.tryCatch(() => subscriber.disconnect(), 'Failed to disconnect'));
// Pure helper functions for getting claimed segments
this.scanForLockKeys = async () => {
const lockKeys = [];
let cursor = '0';
do {
const reply = await this.redis.scan(cursor, {
MATCH: `${this.lockPrefix}*`,
COUNT: 100, // Process in batches of 100
});
cursor = reply.cursor;
lockKeys.push(...reply.keys);
} while (cursor !== '0');
return lockKeys;
};
this.scanForLockKeysTask = () => fp_1.TaskEither.tryCatch(() => this.scanForLockKeys(), 'Failed to scan for lock keys');
this.getLockValues = (lockKeys) => fp_1.TaskEither.tryCatch(() => this.redis.mGet(lockKeys), 'Failed to get lock values');
this.parseLockValue = (lockValue, lockKey) => {
if (!lockValue)
return null;
try {
const data = JSON.parse(lockValue);
const segmentKey = lockKey.replace(this.lockPrefix, '');
return {
segmentKey,
workerId: data.workerId,
claimTime: data.claimedAt,
expiresAt: data.expiresAt,
outputPath: data.outputPath || '',
};
}
catch {
return null;
}
};
this.processClaims = (lockKeys, lockValues) => {
const claimedSegments = [];
for (let i = 0; i < lockKeys.length; i++) {
const parsed = this.parseLockValue(lockValues[i], lockKeys[i]);
if (parsed) {
claimedSegments.push(parsed);
}
}
return claimedSegments;
};
// Pure helper functions for force claiming
this.buildForceClaimScript = () => `
local lockKey = KEYS[1]
local completedKey = KEYS[2]
local newWorkerId = ARGV[1]
local now = tonumber(ARGV[2])
local expiresAt = tonumber(ARGV[3])
local ttl = ARGV[4]
-- 1. Do not steal if already completed
if redis.call('get', completedKey) then
return 0
end
local lockValue = redis.call('get', lockKey)
if not lockValue then
return 0 -- Lock was released, not stuck
end
local data = cjson.decode(lockValue)
-- 2. Do not steal if already stolen by another worker
if data.forced and data.workerId ~= newWorkerId then
return 0
end
-- 3. Do not steal if the lock has not expired (i.e., not stuck)
if tonumber(data.expiresAt) > now then
return 0
end
-- 4. Atomically steal the lock
local newLockData = {
workerId = newWorkerId,
claimedAt = now,
expiresAt = expiresAt,
forced = true,
originalWorkerId = data.workerId,
outputPath = data.outputPath or ARGV[5] or ''
}
redis.call('set', lockKey, cjson.encode(newLockData), 'PX', ttl)
return 1
`;
this.buildLockKey = (segmentKey) => this.lockPrefix + segmentKey;
this.buildCompletedKey = (segmentKey) => this.completedPrefix + segmentKey;
this.executeForceClaimScript = (lockKey, completedKey, newWorkerId, now, expiresAt, outputPath) => fp_1.TaskEither.tryCatch(() => this.redis.eval(this.buildForceClaimScript(), {
keys: [lockKey, completedKey],
arguments: [
newWorkerId,
now.toString(),
expiresAt.toString(),
this.defaultTTL.toString(),
outputPath,
],
}), 'Failed to execute force claim script')
.map((result) => result === 1);
// Input validation methods
this.isValidWorkerId = (workerId) => {
if (!workerId || typeof workerId !== 'string') {
return false;
}
// Allow alphanumeric, hyphens, underscores, max 50 characters
return /^[a-zA-Z0-9_-]{1,50}$/.test(workerId);
};
this.validateAndSanitizeOutputPath = (outputPath) => {
if (!outputPath) {
return '';
}
if (typeof outputPath !== 'string') {
throw new Error('Output path must be a string');
}
// Remove any potential injection characters and limit length
let sanitized = outputPath
.replace(/["'\\]/g, '') // Remove quotes and backslashes
.trim()
.substring(0, 500); // Limit to 500 characters
// Remove control characters using char code filtering
sanitized = sanitized
.split('')
.filter(char => {
const code = char.charCodeAt(0);
return code >= 32 && code <= 126; // Only printable ASCII
})
.join('');
if (sanitized.length === 0 && outputPath.length > 0) {
throw new Error('Output path contains only invalid characters');
}
return sanitized;
};
// Default to 7 days if not specified
this.completedSegmentTTL = completedSegmentTTL || 7 * 24 * 60 * 60 * 1000;
// Validate workerId on construction
if (!this.isValidWorkerId(workerId)) {
throw new Error(`Invalid workerId: ${workerId}. Must be alphanumeric with hyphens/underscores, max 50 chars`);
}
}
/**
* Try to claim a segment for processing
*/
async claimSegment(fileId, streamType, quality, streamIndex, segmentIndex, outputPath) {
const segmentKey = this.getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex);
const lockKey = `${this.lockPrefix}${segmentKey}`;
const now = Date.now();
const expiresAt = now + this.defaultTTL;
// Validate inputs before storing
const validatedOutputPath = this.validateAndSanitizeOutputPath(outputPath);
// Try to acquire lock atomically
const acquired = await this.redis.set(lockKey, JSON.stringify({
workerId: this.workerId,
claimedAt: now,
expiresAt,
outputPath: validatedOutputPath,
}), { NX: true,
PX: this.defaultTTL });
if (!acquired) {
return this.createFailedClaim(segmentKey);
}
// Mark segment as processing
await this.redis.set(`${this.statusPrefix}${segmentKey}`, 'processing', { PX: this.defaultTTL * 2 });
return this.createSuccessfulClaim(segmentKey, lockKey, expiresAt);
}
/**
* Check if a segment is already completed
*/
async isSegmentCompleted(fileId, streamType, quality, streamIndex, segmentIndex) {
const segmentKey = this.getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex);
const completed = await this.redis.get(`${this.completedPrefix}${segmentKey}`);
return completed === 'true';
}
/**
* Mark a segment as completed
*/
async markSegmentCompleted(fileId, streamType, quality, streamIndex, segmentIndex) {
const segmentKey = this.getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex);
// Set completed status with configurable TTL
await this.redis.set(`${this.completedPrefix}${segmentKey}`, 'true', { PX: this.completedSegmentTTL });
// Update status
await this.redis.set(`${this.statusPrefix}${segmentKey}`, 'completed', { PX: this.completedSegmentTTL });
}
/**
* Get the status of a segment
*/
async getSegmentStatus(fileId, streamType, quality, streamIndex, segmentIndex) {
const segmentKey = this.getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex);
return await this.redis.get(`${this.statusPrefix}${segmentKey}`);
}
/**
* Publish segment completion event
*/
async publishSegmentComplete(fileId, streamType, quality, streamIndex, segmentIndex) {
const segmentKey = this.getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex);
const channel = `transcoder:segment:complete:${segmentKey}`;
await this.redis.publish(channel, 'completed');
}
/**
* Subscribe to segment completion events
*/
async subscribeToSegmentComplete(fileId, streamType, quality, streamIndex, segmentIndex, callback) {
const segmentKey = this.getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex);
const channel = this.createCompletionChannel(segmentKey);
const subscriber = this.redis.duplicate();
return this.connectSubscriber(subscriber)
.chain(() => this.subscribeToChannel(subscriber, channel, callback))
.map(() => this.createUnsubscribeFunction(subscriber, channel))
.orElse((error) => {
// Ensure we disconnect on any failure to prevent leaks
return this.disconnectSubscriber(subscriber)
.chain(() => fp_1.TaskEither.error(error));
})
.toPromise();
}
getSegmentKey(fileId, streamType, quality, streamIndex, segmentIndex) {
return `${fileId}:${streamType}:${quality}:${streamIndex}:${segmentIndex}`;
}
createFailedClaim(segmentKey) {
return {
acquired: false,
segmentKey,
workerId: this.workerId,
expiresAt: 0,
extend: async () => false,
release: async () => { },
};
}
createSuccessfulClaim(segmentKey, lockKey, expiresAt) {
return {
acquired: true,
segmentKey,
workerId: this.workerId,
expiresAt,
extend: async () => {
// Extend lock using Lua script for atomicity
const script = `
local lock = redis.call('get', KEYS[1])
if lock then
local data = cjson.decode(lock)
if data.workerId == ARGV[1] then
local newExpiry = tonumber(ARGV[2])
data.expiresAt = newExpiry
redis.call('set', KEYS[1], cjson.encode(data), 'PX', ARGV[3])
return 1
end
end
return 0
`;
const newExpiresAt = Date.now() + this.defaultTTL;
const result = await this.redis.eval(script, {
keys: [lockKey],
arguments: [this.workerId, newExpiresAt.toString(), this.defaultTTL.toString()],
});
return result === 1;
},
release: async () => {
// Release lock only if we own it
const script = `
local lock = redis.call('get', KEYS[1])
if lock then
local data = cjson.decode(lock)
if data.workerId == ARGV[1] then
return redis.call('del', KEYS[1])
end
end
return 0
`;
await this.redis.eval(script, {
keys: [lockKey],
arguments: [this.workerId],
});
},
};
}
/**
* Get all currently claimed segments with their metadata
* Used for work stealing detection
*/
async getClaimedSegments() {
return this.scanForLockKeysTask()
.filter((keys) => keys.length > 0, () => (0, fp_1.createInternalError)('No lock keys found'))
.chain((lockKeys) => this.getLockValues(lockKeys)
.map((lockValues) => this.processClaims(lockKeys, lockValues)))
.orElse((error) => {
console.error('Failed to get claimed segments:', error);
return fp_1.TaskEither.of([]);
})
.toPromise();
}
/**
* Force claim a segment, overriding existing claims
* Used for work stealing when a worker is unresponsive
*/
async forceClaimSegment(segmentKey, newWorkerId, outputPath) {
const lockKey = this.buildLockKey(segmentKey);
const completedKey = this.buildCompletedKey(segmentKey);
const now = Date.now();
const expiresAt = now + this.defaultTTL;
return this.executeForceClaimScript(lockKey, completedKey, newWorkerId, now, expiresAt, outputPath || '')
.orElse((error) => {
console.error(`Failed to force claim segment ${segmentKey}:`, error);
return fp_1.TaskEither.of(false);
})
.toPromise();
}
/**
* Parse a segment key to extract its components
*/
parseSegmentKey(segmentKey) {
const parts = segmentKey.split(':');
if (parts.length < 5) {
return null;
}
const [fileId, streamType, quality, streamIndex, segmentIndex] = parts;
return {
fileId,
streamType,
quality,
streamIndex: parseInt(streamIndex, 10),
segmentIndex: parseInt(segmentIndex, 10),
};
}
}
exports.RedisSegmentClaimManager = RedisSegmentClaimManager;
//# sourceMappingURL=redisSegmentClaimManager.js.map