@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
408 lines (407 loc) • 17.1 kB
TypeScript
/**
* Video Processor
*
* Handles downloading, validating, and processing video files for AI consumption.
* Since LLMs cannot process raw video, this processor extracts:
* - Structured metadata (duration, resolution, codecs, etc.)
* - Keyframes at configurable intervals (resized to 768px JPEG)
* - Embedded subtitle tracks (if present)
*
* The extracted content is formatted as text + images that can be sent to any
* AI provider for analysis.
*
* Uses mediabunny (pure TypeScript) for metadata extraction, with fluent-ffmpeg
* as a fallback for unsupported formats. Requires ffmpeg for keyframe/subtitle
* extraction (via ffmpeg-static or system PATH).
*
* Key features:
* - Adaptive keyframe extraction intervals based on video duration
* - Frame count capping (max 20 frames) to control token usage
* - JPEG quality optimization for AI vision models
* - Embedded subtitle extraction (SRT format)
* - Graceful degradation on corrupt files or missing codecs
* - Temp file cleanup with finally blocks
* - Configurable timeouts for ffmpeg and ffprobe operations
*
* @module processors/media/VideoProcessor
*
* @example
* ```typescript
* import { videoProcessor, processVideo, isVideoFile } from "./VideoProcessor.js";
*
* // Check if a file is a video file
* if (isVideoFile(fileInfo.mimetype, fileInfo.name)) {
* const result = await processVideo(fileInfo, {
* authHeaders: { Authorization: "Bearer token" },
* });
*
* if (result.success) {
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
* console.log(`Keyframes: ${result.data.frameCount}`);
* console.log(`Text for LLM:\n${result.data.textContent}`);
* }
* }
* ```
*/
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
import type { FileInfo, ProcessedVideo, ProcessorFileProcessingResult, ProcessOptions, VideoProcessorOptions } from "../../types/index.js";
/**
* Narrow a loaded `fluent-ffmpeg` export to the shape this file actually uses:
* a callable carrying the `ffprobe` and `setFfmpegPath` statics.
*
* `tryImport` proves only that the package RESOLVES. Callers invoke the export
* and reach straight for its statics, so a package whose shape changed (ESM
* rewrite, major bump, a shim in node_modules) would otherwise surface as
* "Cannot read properties of undefined (reading 'ffprobe')" from inside
* probeVideo — blaming the call site instead of the package that is wrong.
*/
export declare function assertFluentFfmpegShape(mod: unknown): asserts mod is typeof import("fluent-ffmpeg");
/**
* Video Processor - extracts metadata, keyframes, and subtitles from video files.
*
* Since LLMs cannot process raw video, this processor converts videos into
* a structured representation consisting of:
* 1. Text metadata block (duration, resolution, codecs, etc.)
* 2. Keyframe images (JPEG, resized to 768px max dimension)
* 3. Subtitle text (if embedded in the video)
*
* The processor uses a temp file approach because ffmpeg requires file paths
* for most operations. Temp files are always cleaned up in finally blocks.
*
* @example
* ```typescript
* const processor = new VideoProcessor();
* const result = await processor.processFile({
* id: "video-1",
* name: "presentation.mp4",
* mimetype: "video/mp4",
* size: 15_000_000,
* buffer: videoBuffer,
* });
*
* if (result.success) {
* // result.data.textContent - text description for LLM
* // result.data.keyframes - array of JPEG buffers
* // result.data.subtitleText - extracted subtitles (if any)
* }
* ```
*/
export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
constructor();
/**
* Build processed result stub.
* This is a synchronous placeholder - actual processing happens in the
* overridden processFile method since ffmpeg operations are asynchronous
* and require temp file I/O.
*
* @param buffer - Downloaded file content
* @param fileInfo - Original file information
* @returns Empty ProcessedVideo structure
*/
protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedVideo;
/**
* Override processFile for async video processing with ffmpeg.
*
* Processing pipeline:
* 1. Validate file type and size
* 2. Get buffer (from fileInfo.buffer or download from URL)
* 3. Write buffer to temp file (ffmpeg requires file paths)
* 4. Extract metadata using ffprobe
* 5. Extract keyframes at calculated intervals, resize with sharp
* 6. Extract subtitle tracks if embedded
* 7. Build textContent summary for LLM
* 8. Clean up temp files
*
* @param fileInfo - File information with URL or buffer
* @param options - Optional processing options
* @returns Processing result with extracted video data or error
*/
processFile(fileInfo: FileInfo, options?: ProcessOptions & VideoProcessorOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;
/**
* Probe a video file to extract metadata using ffprobe.
*
* @param filePath - Path to the video file
* @returns Success result with probe data or error message
*/
private probeVideo;
/**
* Probe a video file using mediabunny (pure TypeScript, no native binary).
* Falls back to ffprobe if mediabunny fails or doesn't support the format.
*/
private probeVideoWithMediabunny;
/**
* Read metadata from the ffmpeg binary itself, for hosts that have ffmpeg
* but no ffprobe (mediabunny cannot read AVI, FLV or WMV, and ffprobe was the
* only other reader).
*
* ffmpeg prints the input's streams to stderr before it opens the output, so
* a zero-length run to the null muxer is enough to read them.
*/
private probeVideoWithFfmpeg;
/**
* Parse the input section of an ffmpeg banner: `Duration:`, `bitrate:` and
* one `Stream #…: Video|Audio|Subtitle:` line per stream. Returns undefined
* when there is no positive duration, since frames cannot be timed without it.
*/
private parseFfmpegInputReport;
/** ffmpeg names a layout ("mono", "stereo", "5.1(side)") or counts ("6 channels"). */
private channelCountFromLayout;
/**
* Build a structured metadata object from ffprobe data.
*
* @param probeData - Raw ffprobe output
* @param fileSize - Original file size in bytes
* @returns Structured video metadata
*/
private buildMetadata;
/**
* Clamp a caller-supplied frame quality into sharp's valid 1-100 range,
* falling back to the default when absent or non-numeric (#478).
*/
private static resolveFrameQuality;
/**
* Extract the clip's audio track and transcribe the speech in it (#433).
*
* ## Why this exists at all, now that Gemini hears video directly
*
* It does not help Gemini — a native provider receives the clip and its
* audio together. It is for everyone else. A provider on the frame path
* gets stills and nothing else, so a recorded standup, a support call or a
* narrated demo arrives with its entire spoken content missing, and
* "summarise this meeting" is answered from four screenshots.
*
* ## Why it is not a stub
*
* The issue asked for one, on the grounds that no transcription backend
* existed yet. One does: `AudioProcessor` has shipped Whisper transcription
* for standalone audio files for some time. A method that logged "not yet
* implemented" and returned undefined would be dead code sitting next to a
* working implementation of the same thing, and `--transcribe-audio` would
* still do nothing.
*
* The Whisper call is reproduced here rather than shared with
* `AudioProcessor`: extracting it into a common module is the better
* long-term shape, but that file is being edited concurrently, and a
* merge conflict in the audio pipeline is a worse outcome than fifty
* duplicated lines. The duplication is worth removing once both land.
*
* ## Failure behaviour
*
* Best-effort throughout, and never throws: a clip with no audio track, a
* machine without ffmpeg, a missing key, an oversized track or a failed
* request all return a *reason* and leave the rest of the pipeline intact.
* Each reason is distinct, because from the outside every one of them
* looks the same — no transcript — while the remedies differ completely.
*
* @param videoPath - Temp path to the video, already written by the caller
* @param tempDir - Caller-owned temp directory; removed in its `finally`
* @param metadata - Probed metadata, consulted for an audio track
* @param filename - Display name, for logs
* @param options - Caller settings; transcription runs only when asked
*/
private extractAndTranscribeAudio;
/**
* Extract keyframes from a video at calculated intervals.
*
* The interval between frames is determined by the video duration:
* - <= 10s: every 1s (very short clips — dense coverage)
* - <= 30s: every 2s (short bug clips)
* - <= 120s: every 5s (standard screen recordings)
* - <= 600s: every 15s (longer demos)
* - <= 1800s: every 60s (meeting recordings)
* - > 1800s: every 180s (full meetings)
*
* Results are capped at MAX_FRAMES (100) and each frame is resized
* to fit within 768x768px while maintaining aspect ratio.
* The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
* the interval widens to duration/MAX_FRAMES for full-video coverage.
*
* A caller-supplied `options.frames` overrides the tier schedule entirely:
* that many frames are spread evenly across the clip, still capped at
* MAX_FRAMES. `options.quality` and `options.format` reach the encoder (#478).
*
* @param videoPath - Path to the video file
* @param tempDir - Temp directory for frame output
* @param durationSec - Video duration in seconds
* @param options - Caller frame budget / encoder settings
* @returns Each kept frame with the second it was sampled at, ascending
*/
private extractKeyframes;
/**
* Run ffmpeg to extract frames at specified timestamps.
*
* Uses the `-vf select` filter to pick frames at least `intervalSec` apart,
* which is more efficient than seeking for each frame individually. This is
* a minimum-gap filter, not a seek to an exact time: on a source whose real
* frame spacing exceeds the requested interval, the frame actually selected
* can land well after the ideal schedule. A `showinfo` stage between
* `select` and `scale` reports each selected frame's true `pts_time` to
* stderr so the caller can label frames with when they were really
* sampled, not the idealized request.
*
* @param videoPath - Path to the video file
* @param outputDir - Directory to write frame files
* @param timestamps - Array of timestamps in seconds
* @param intervalSec - Minimum gap enforced between selected frames
* @returns The real `pts_time` (seconds) ffmpeg reported for each selected
* frame, in output-file order; may be shorter than `timestamps` if fewer
* real frames were available to select
*/
private runFfmpegFrameExtraction;
private runFfmpegFrameExtractionAttempt;
/**
* Determine the frame extraction interval based on video duration.
*
* @param durationSec - Video duration in seconds
* @returns Interval in seconds between extracted frames
*/
private getFrameInterval;
/**
* Extract embedded subtitle text from the first subtitle track.
*
* Uses ffmpeg to convert the first subtitle stream to SRT format,
* then strips SRT formatting (timestamps, sequence numbers) to produce
* plain text.
*
* @param videoPath - Path to the video file
* @param tempDir - Temp directory for subtitle output
* @returns Extracted subtitle text, or undefined if extraction fails
*/
private extractSubtitles;
/**
* Parse SRT subtitle content into plain text.
* Strips sequence numbers, timestamps, and blank lines.
*
* @param srt - Raw SRT content
* @returns Plain text from subtitles
*/
private parseSrtToPlainText;
/**
* Build a structured text description of the video for LLM consumption.
*
* The output includes:
* - File name and basic info
* - Technical metadata (resolution, codec, duration, etc.)
* - Frame extraction summary
* - Subtitle text (if available)
*
* @param metadata - Extracted video metadata
* @param keyframeTimestampsSec - The real second each kept keyframe was sampled at
* @param subtitleText - Extracted subtitle text (if any)
* @param filename - Original filename
* @param transcript - Machine transcription of the spoken audio (if requested and available)
* @returns Formatted text content for the LLM
*/
private buildTextContent;
/**
* Format a duration in seconds to a human-readable string.
*
* Delegates to the shared formatter so audio and video agree — see
* `formatMediaDuration`. Rounding replaces the previous truncation, so a
* 2.6s clip now reads "3s" rather than "2s" and matches what the audio
* side reports for the same stream.
*
* @param seconds - Duration in seconds
* @returns Formatted string (e.g., "1h 23m 45s")
*/
private formatDuration;
/**
* Get a file extension from FileInfo, falling back to ".mp4".
*
* @param fileInfo - File information
* @returns File extension with leading dot
*/
private getExtensionFromFileInfo;
/**
* Write a buffer to a file using streaming to handle large files efficiently.
*
* @param buffer - Buffer to write
* @param filePath - Destination file path
*/
private writeBufferToFile;
/**
* Extract frames from a specific time range in a video.
*
* This is the on-demand extraction method called by the `extract_file_content`
* tool. Unlike initial keyframe extraction (which covers the full video),
* this targets a specific time window with configurable frame count.
*
* @param buffer - Video file buffer
* @param filename - Original filename (for extension detection)
* @param startSec - Start time in seconds
* @param endSec - End time in seconds
* @param frameCount - Number of frames to extract in the range (default: 5)
* @returns Array of JPEG frame buffers
*/
extractFrameRange(buffer: Buffer, filename: string, startSec: number, endSec: number, frameCount?: number): Promise<Buffer[]>;
/**
* Guess file extension from filename, with fallback to .mp4.
*/
private guessExtensionFromName;
}
/**
* Singleton Video processor instance.
* Use this for standard video processing operations.
*
* @example
* ```typescript
* import { videoProcessor } from "./VideoProcessor.js";
*
* const result = await videoProcessor.processFile(fileInfo);
* ```
*/
export declare const videoProcessor: VideoProcessor;
/**
* Check if a file is a video file.
* Matches by MIME type or file extension.
*
* @param mimetype - MIME type of the file
* @param filename - Filename (for extension-based detection)
* @returns true if the file is a supported video file
*
* @example
* ```typescript
* if (isVideoFile("video/mp4", "recording.mp4")) {
* const result = await processVideo(fileInfo);
* }
*
* if (isVideoFile("", "clip.mkv")) {
* // Also matches by extension
* }
* ```
*/
export declare function isVideoFile(mimetype: string, filename: string): boolean;
/**
* Process a single video file.
* Convenience function that uses the singleton processor.
*
* @param fileInfo - File information (can include URL or buffer)
* @param options - Optional processing options (auth headers, timeout, retry config)
* @returns Processing result with extracted video data or error
*
* @example
* ```typescript
* import { processVideo } from "./VideoProcessor.js";
*
* const result = await processVideo({
* id: "vid-123",
* name: "demo.mp4",
* mimetype: "video/mp4",
* size: 15_000_000,
* buffer: videoBuffer,
* });
*
* if (result.success) {
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
* console.log(`Extracted ${result.data.frameCount} keyframes`);
* console.log(`Text content:\n${result.data.textContent}`);
*
* if (result.data.subtitleText) {
* console.log(`Subtitles:\n${result.data.subtitleText}`);
* }
* } else {
* console.error(`Processing failed: ${result.error?.userMessage}`);
* }
* ```
*/
export declare function processVideo(fileInfo: FileInfo, options?: ProcessOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;