@juspay/neurolink/dist/utils/videoProcessor.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Video Generation Processing Utility
 *
 * Central registry + dispatch for video-generation handlers across
 * providers (Vertex Veo, Kling, Runway, Replicate-hosted models, etc.).
 *
 * Mirrors the static-handler-registry pattern established by
 * `TTSProcessor` (`utils/ttsProcessor.ts`) and `STTProcessor`
 * (`utils/sttProcessor.ts`).
 *
 * @module utils/videoProcessor
 */
import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
import { VIDEO_ERROR_CODES } from "../constants/videoErrors.js";
import { SpanSerializer, SpanStatus, SpanType, getMetricsAggregator, } from "../observability/index.js";
import { logger } from "./logger.js";
import { withTimeout } from "./async/withTimeout.js";
// Video generation is legitimately minutes-long (Kling/Runway render queues),
// so the bound is generous — its job is to convert a wedged handler into an
// error rather than an eternal hang, not to police normal latency.
const VIDEO_GENERATION_TIMEOUT_MS = 600_000;
// VideoError is canonical in vertexVideoHandler.ts (existing). Re-export
// here so consumers of `VideoProcessor` can import the typed error from
// the same module. Both throws and instanceof checks resolve to the same
// class.
import { VideoError } from "../adapters/video/vertexVideoHandler.js";
import { HandlerRegistry } from "../core/handlerRegistry.js";
export { VideoError, VIDEO_ERROR_CODES };
/**
 * Static processor managing the video handler registry.
 *
 * Handlers register themselves during `ProviderRegistry._doRegister()`
 * via `VideoProcessor.registerHandler(name, instance)`. Lookups are
 * O(1) on a normalised lower-case provider key.
 */
export class VideoProcessor {
    static registry = new HandlerRegistry("VideoProcessor");
    /**
     * Register a video handler for a specific provider.
     */
    static registerHandler(providerName, handler) {
        const key = providerName ? providerName.toLowerCase() : providerName;
        this.registry.register(providerName, handler);
        logger.debug(`[VideoProcessor] Registered video handler: ${key}`);
    }
    /**
     * Check if a provider has a registered video handler.
     */
    static supports(providerName) {
        return this.registry.supports(providerName);
    }
    /**
     * List the names of all registered providers.
     */
    static listProviders() {
        return this.registry.list();
    }
    static getHandler(providerName) {
        return this.registry.get(providerName);
    }
    /**
     * Clear all registered handlers (for testing).
     */
    static clearHandlers() {
        this.registry.clear();
    }
    static buildSpanAttributes(provider, options) {
        return {
            "video.operation": "generate",
            "video.provider": provider,
            "video.resolution": options.resolution,
            "video.duration": options.length,
            "video.aspect_ratio": options.aspectRatio,
            "video.audio": options.audio,
        };
    }
    static async generate(provider, optionsOrImage, legacyPrompt, legacyOptions, legacyRegion) {
        const bag = Buffer.isBuffer(optionsOrImage)
            ? {
                image: optionsOrImage,
                prompt: legacyPrompt ?? "",
                ...(legacyRegion !== undefined ? { region: legacyRegion } : {}),
                ...(legacyOptions ?? {}),
            }
            : optionsOrImage;
        const { image, prompt, region, ...videoOptions } = bag;
        // A fired timeout must also cancel the handler's own request/polling
        // loop — otherwise the caller sees the rejection while a ghost
        // generation keeps polling (and possibly billing) for the rest of the
        // render. Chain the internal controller onto any caller-supplied
        // signal so both cancellation sources reach the handler.
        const timeoutAbort = new AbortController();
        const abortSignal = videoOptions.abortSignal
            ? AbortSignal.any([videoOptions.abortSignal, timeoutAbort.signal])
            : timeoutAbort.signal;
        const handlerOptions = { ...videoOptions, abortSignal };
        const span = SpanSerializer.createSpan(SpanType.MEDIA_GENERATION, "video.generate", this.buildSpanAttributes(provider, videoOptions));
        try {
            const handler = this.getHandler(provider);
            if (!handler) {
                throw new VideoError({
                    code: VIDEO_ERROR_CODES.PROVIDER_NOT_SUPPORTED,
                    message: `Video provider "${provider}" is not registered. Available: ${this.listProviders().join(", ")}`,
                    category: ErrorCategory.CONFIGURATION,
                    severity: ErrorSeverity.HIGH,
                    retriable: false,
                    context: { provider, available: this.listProviders() },
                });
            }
            if (!handler.isConfigured()) {
                throw new VideoError({
                    code: VIDEO_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
                    message: `Video provider "${provider}" is not configured. Set the required credentials.`,
                    category: ErrorCategory.CONFIGURATION,
                    severity: ErrorSeverity.HIGH,
                    retriable: false,
                    context: { provider },
                });
            }
            logger.debug(`[VideoProcessor] Starting video generation with provider: ${provider}`);
            // Bounded per repo guideline (async provider calls wrap withTimeout):
            // video generation is legitimately slow, so the deadline is generous —
            // but a wedged handler must error, never hang the caller forever.
            const result = await withTimeout(handler.generate(image, prompt, handlerOptions, region), VIDEO_GENERATION_TIMEOUT_MS, `Video generation via "${provider}" timed out after ${VIDEO_GENERATION_TIMEOUT_MS}ms`);
            const ended = SpanSerializer.endSpan(span, SpanStatus.OK);
            getMetricsAggregator().recordSpan(ended);
            logger.info(`[VideoProcessor] Generated ${result.data.length} bytes (${provider})`);
            return result;
        }
        catch (err) {
            // Cancel the ghost: on timeout the handler promise is still pending;
            // aborting here stops its polling loop. On handler-originated errors
            // the promise has already settled, so the abort is a no-op.
            timeoutAbort.abort();
            const ended = SpanSerializer.endSpan(span, SpanStatus.ERROR, err instanceof Error ? err.message : String(err));
            getMetricsAggregator().recordSpan(ended);
            if (err instanceof VideoError) {
                throw err;
            }
            const message = err instanceof Error ? err.message : String(err);
            throw new VideoError({
                code: VIDEO_ERROR_CODES.GENERATION_FAILED,
                message: `Video generation failed for provider "${provider}": ${message}`,
                category: ErrorCategory.EXECUTION,
                severity: ErrorSeverity.HIGH,
                retriable: true,
                context: { provider, options: videoOptions, region },
                originalError: err instanceof Error ? err : undefined,
            });
        }
    }
    /**
     * Generate a transition clip via the registered handler (Director Mode).
     *
     * Providers without first-and-last-frame interpolation surface a typed
     * `TRANSITION_NOT_SUPPORTED` error here; callers should fall back to
     * generating a regular clip with a transition prompt.
     */
    static async generateTransition(provider, firstFrame, lastFrame, prompt, options, region) {
        const handler = this.getHandler(provider);
        if (!handler) {
            throw new VideoError({
                code: VIDEO_ERROR_CODES.PROVIDER_NOT_SUPPORTED,
                message: `Video provider "${provider}" is not registered for transitions`,
                category: ErrorCategory.CONFIGURATION,
                severity: ErrorSeverity.HIGH,
                retriable: false,
                context: { provider, available: this.listProviders() },
            });
        }
        if (!handler.generateTransition) {
            throw new VideoError({
                code: VIDEO_ERROR_CODES.TRANSITION_NOT_SUPPORTED,
                message: `Video provider "${provider}" does not support transition clips`,
                category: ErrorCategory.VALIDATION,
                severity: ErrorSeverity.MEDIUM,
                retriable: false,
                context: { provider },
            });
        }
        if (!handler.isConfigured()) {
            throw new VideoError({
                code: VIDEO_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
                message: `Video provider "${provider}" is not configured`,
                category: ErrorCategory.CONFIGURATION,
                severity: ErrorSeverity.HIGH,
                retriable: false,
                context: { provider },
            });
        }
        // Same ghost-cancellation contract as generate(): a fired timeout
        // aborts the handler's polling loop, chained onto any caller signal.
        const timeoutAbort = new AbortController();
        const abortSignal = options?.abortSignal
            ? AbortSignal.any([options.abortSignal, timeoutAbort.signal])
            : timeoutAbort.signal;
        const handlerOptions = {
            ...(options ?? {}),
            abortSignal,
        };
        try {
            // Same bound as generate(): a wedged transition must error, not hang.
            return await withTimeout(handler.generateTransition(firstFrame, lastFrame, prompt, handlerOptions, region), VIDEO_GENERATION_TIMEOUT_MS, `Video transition via "${provider}" timed out after ${VIDEO_GENERATION_TIMEOUT_MS}ms`);
        }
        catch (err) {
            timeoutAbort.abort();
            if (err instanceof VideoError) {
                throw err;
            }
            const message = err instanceof Error ? err.message : String(err);
            throw new VideoError({
                code: VIDEO_ERROR_CODES.DIRECTOR_TRANSITION_FAILED,
                message: `Video transition generation failed for provider "${provider}": ${message}`,
                category: ErrorCategory.EXECUTION,
                severity: ErrorSeverity.MEDIUM,
                retriable: true,
                context: {
                    provider,
                    firstFrameSize: firstFrame.length,
                    lastFrameSize: lastFrame.length,
                    durationSeconds: options?.durationSeconds,
                },
                originalError: err instanceof Error ? err : undefined,
            });
        }
    }
}