@juspay/neurolink
Version:
Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio
312 lines (311 loc) • 11.6 kB
TypeScript
/**
* Video Processor
*
* Handles downloading, validating, and processing video files for AI consumption.
* Since LLMs cannot process raw video, this processor extracts:
* - Structured metadata (duration, resolution, codecs, etc.)
* - Keyframes at configurable intervals (resized to 768px JPEG)
* - Embedded subtitle tracks (if present)
*
* The extracted content is formatted as text + images that can be sent to any
* AI provider for analysis.
*
* Uses mediabunny (pure TypeScript) for metadata extraction, with fluent-ffmpeg
* as a fallback for unsupported formats. Requires ffmpeg for keyframe/subtitle
* extraction (via ffmpeg-static or system PATH).
*
* Key features:
* - Adaptive keyframe extraction intervals based on video duration
* - Frame count capping (max 20 frames) to control token usage
* - JPEG quality optimization for AI vision models
* - Embedded subtitle extraction (SRT format)
* - Graceful degradation on corrupt files or missing codecs
* - Temp file cleanup with finally blocks
* - Configurable timeouts for ffmpeg and ffprobe operations
*
* @module processors/media/VideoProcessor
*
* @example
* ```typescript
* import { videoProcessor, processVideo, isVideoFile } from "./VideoProcessor.js";
*
* // Check if a file is a video file
* if (isVideoFile(fileInfo.mimetype, fileInfo.name)) {
* const result = await processVideo(fileInfo, {
* authHeaders: { Authorization: "Bearer token" },
* });
*
* if (result.success) {
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
* console.log(`Keyframes: ${result.data.frameCount}`);
* console.log(`Text for LLM:\n${result.data.textContent}`);
* }
* }
* ```
*/
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
import type { FileInfo, ProcessedVideo, ProcessorFileProcessingResult, ProcessOptions } from "../../types/index.js";
/**
* Video Processor - extracts metadata, keyframes, and subtitles from video files.
*
* Since LLMs cannot process raw video, this processor converts videos into
* a structured representation consisting of:
* 1. Text metadata block (duration, resolution, codecs, etc.)
* 2. Keyframe images (JPEG, resized to 768px max dimension)
* 3. Subtitle text (if embedded in the video)
*
* The processor uses a temp file approach because ffmpeg requires file paths
* for most operations. Temp files are always cleaned up in finally blocks.
*
* @example
* ```typescript
* const processor = new VideoProcessor();
* const result = await processor.processFile({
* id: "video-1",
* name: "presentation.mp4",
* mimetype: "video/mp4",
* size: 15_000_000,
* buffer: videoBuffer,
* });
*
* if (result.success) {
* // result.data.textContent - text description for LLM
* // result.data.keyframes - array of JPEG buffers
* // result.data.subtitleText - extracted subtitles (if any)
* }
* ```
*/
export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
constructor();
/**
* Build processed result stub.
* This is a synchronous placeholder - actual processing happens in the
* overridden processFile method since ffmpeg operations are asynchronous
* and require temp file I/O.
*
* @param buffer - Downloaded file content
* @param fileInfo - Original file information
* @returns Empty ProcessedVideo structure
*/
protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedVideo;
/**
* Override processFile for async video processing with ffmpeg.
*
* Processing pipeline:
* 1. Validate file type and size
* 2. Get buffer (from fileInfo.buffer or download from URL)
* 3. Write buffer to temp file (ffmpeg requires file paths)
* 4. Extract metadata using ffprobe
* 5. Extract keyframes at calculated intervals, resize with sharp
* 6. Extract subtitle tracks if embedded
* 7. Build textContent summary for LLM
* 8. Clean up temp files
*
* @param fileInfo - File information with URL or buffer
* @param options - Optional processing options
* @returns Processing result with extracted video data or error
*/
processFile(fileInfo: FileInfo, options?: ProcessOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;
/**
* Probe a video file to extract metadata using ffprobe.
*
* @param filePath - Path to the video file
* @returns Success result with probe data or error message
*/
private probeVideo;
/**
* Probe a video file using mediabunny (pure TypeScript, no native binary).
* Falls back to ffprobe if mediabunny fails or doesn't support the format.
*/
private probeVideoWithMediabunny;
/**
* Build a structured metadata object from ffprobe data.
*
* @param probeData - Raw ffprobe output
* @param fileSize - Original file size in bytes
* @returns Structured video metadata
*/
private buildMetadata;
/**
* Extract keyframes from a video at calculated intervals.
*
* The interval between frames is determined by the video duration:
* - <= 10s: every 1s (very short clips — dense coverage)
* - <= 30s: every 2s (short bug clips)
* - <= 120s: every 5s (standard screen recordings)
* - <= 600s: every 15s (longer demos)
* - <= 1800s: every 60s (meeting recordings)
* - > 1800s: every 180s (full meetings)
*
* Results are capped at MAX_FRAMES (100) and each frame is resized
* to fit within 768x768px while maintaining aspect ratio.
* The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
* the interval widens to duration/MAX_FRAMES for full-video coverage.
*
* @param videoPath - Path to the video file
* @param tempDir - Temp directory for frame output
* @param durationSec - Video duration in seconds
* @returns Array of JPEG frame buffers
*/
private extractKeyframes;
/**
* Run ffmpeg to extract frames at specified timestamps.
*
* Uses the `-vf select` filter to pick frames at exact timestamps,
* which is more efficient than seeking for each frame individually.
*
* @param videoPath - Path to the video file
* @param outputDir - Directory to write frame files
* @param timestamps - Array of timestamps in seconds
*/
private runFfmpegFrameExtraction;
/**
* Determine the frame extraction interval based on video duration.
*
* @param durationSec - Video duration in seconds
* @returns Interval in seconds between extracted frames
*/
private getFrameInterval;
/**
* Extract embedded subtitle text from the first subtitle track.
*
* Uses ffmpeg to convert the first subtitle stream to SRT format,
* then strips SRT formatting (timestamps, sequence numbers) to produce
* plain text.
*
* @param videoPath - Path to the video file
* @param tempDir - Temp directory for subtitle output
* @returns Extracted subtitle text, or undefined if extraction fails
*/
private extractSubtitles;
/**
* Parse SRT subtitle content into plain text.
* Strips sequence numbers, timestamps, and blank lines.
*
* @param srt - Raw SRT content
* @returns Plain text from subtitles
*/
private parseSrtToPlainText;
/**
* Build a structured text description of the video for LLM consumption.
*
* The output includes:
* - File name and basic info
* - Technical metadata (resolution, codec, duration, etc.)
* - Frame extraction summary
* - Subtitle text (if available)
*
* @param metadata - Extracted video metadata
* @param frameCount - Number of keyframes extracted
* @param subtitleText - Extracted subtitle text (if any)
* @param filename - Original filename
* @returns Formatted text content for the LLM
*/
private buildTextContent;
/**
* Format a duration in seconds to a human-readable string.
*
* @param seconds - Duration in seconds
* @returns Formatted string (e.g., "1h 23m 45s")
*/
private formatDuration;
/**
* Get a file extension from FileInfo, falling back to ".mp4".
*
* @param fileInfo - File information
* @returns File extension with leading dot
*/
private getExtensionFromFileInfo;
/**
* Write a buffer to a file using streaming to handle large files efficiently.
*
* @param buffer - Buffer to write
* @param filePath - Destination file path
*/
private writeBufferToFile;
/**
* Extract frames from a specific time range in a video.
*
* This is the on-demand extraction method called by the `extract_file_content`
* tool. Unlike initial keyframe extraction (which covers the full video),
* this targets a specific time window with configurable frame count.
*
* @param buffer - Video file buffer
* @param filename - Original filename (for extension detection)
* @param startSec - Start time in seconds
* @param endSec - End time in seconds
* @param frameCount - Number of frames to extract in the range (default: 5)
* @returns Array of JPEG frame buffers
*/
extractFrameRange(buffer: Buffer, filename: string, startSec: number, endSec: number, frameCount?: number): Promise<Buffer[]>;
/**
* Guess file extension from filename, with fallback to .mp4.
*/
private guessExtensionFromName;
}
/**
* Singleton Video processor instance.
* Use this for standard video processing operations.
*
* @example
* ```typescript
* import { videoProcessor } from "./VideoProcessor.js";
*
* const result = await videoProcessor.processFile(fileInfo);
* ```
*/
export declare const videoProcessor: VideoProcessor;
/**
* Check if a file is a video file.
* Matches by MIME type or file extension.
*
* @param mimetype - MIME type of the file
* @param filename - Filename (for extension-based detection)
* @returns true if the file is a supported video file
*
* @example
* ```typescript
* if (isVideoFile("video/mp4", "recording.mp4")) {
* const result = await processVideo(fileInfo);
* }
*
* if (isVideoFile("", "clip.mkv")) {
* // Also matches by extension
* }
* ```
*/
export declare function isVideoFile(mimetype: string, filename: string): boolean;
/**
* Process a single video file.
* Convenience function that uses the singleton processor.
*
* @param fileInfo - File information (can include URL or buffer)
* @param options - Optional processing options (auth headers, timeout, retry config)
* @returns Processing result with extracted video data or error
*
* @example
* ```typescript
* import { processVideo } from "./VideoProcessor.js";
*
* const result = await processVideo({
* id: "vid-123",
* name: "demo.mp4",
* mimetype: "video/mp4",
* size: 15_000_000,
* buffer: videoBuffer,
* });
*
* if (result.success) {
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
* console.log(`Extracted ${result.data.frameCount} keyframes`);
* console.log(`Text content:\n${result.data.textContent}`);
*
* if (result.data.subtitleText) {
* console.log(`Subtitles:\n${result.data.subtitleText}`);
* }
* } else {
* console.error(`Processing failed: ${result.error?.userMessage}`);
* }
* ```
*/
export declare function processVideo(fileInfo: FileInfo, options?: ProcessOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;