ai/src/generate-video/generate-video.ts

AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.

import type {
  Experimental_VideoModelV4,
  Experimental_VideoModelV4CallOptions,
  Experimental_VideoModelV4File,
  Experimental_VideoModelV4FrameImage,
  Experimental_VideoModelV4FrameType,
  SharedV4ProviderMetadata,
} from '@ai-sdk/provider';
import {
  convertBase64ToUint8Array,
  withUserAgentSuffix,
  type DataContent,
  detectMediaType,
  type ProviderOptions,
} from '@ai-sdk/provider-utils';
import { NoVideoGeneratedError } from '../error/no-video-generated-error';
import {
  DefaultGeneratedFile,
  type GeneratedFile,
} from '../generate-text/generated-file';
import { logWarnings } from '../logger/log-warnings';
import { resolveVideoModel } from '../model/resolve-model';
import type { VideoModel } from '../types/video-model';
import type { VideoModelResponseMetadata } from '../types/video-model-response-metadata';
import type { Warning } from '../types/warning';
import { createDownload } from '../util/download/create-download';
import { prepareRetries } from '../util/prepare-retries';
import { VERSION } from '../version';
import type { GenerateVideoResult } from './generate-video-result';
import { splitDataUrl } from '../prompt/split-data-url';

export type GenerateVideoPrompt =
  | string
  | {
      image: DataContent;
      text?: string;
    };

/**
 * Generates videos using a video model.
 *
 * @param model - The video model to use.
 * @param prompt - The prompt that should be used to generate the video.
 * @param n - Number of videos to generate. Default: 1.
 * @param aspectRatio - Aspect ratio of the videos to generate. Must have the format `{width}:{height}`.
 * @param resolution - Resolution of the videos to generate. Must have the format `{width}x{height}`.
 * @param duration - Duration of the video in seconds.
 * @param fps - Frames per second for the video.
 * @param seed - Seed for the video generation.
 * @param frameImages - Role-tagged image inputs for image-to-video and first-last-frame generation.
 * @param inputReferences - Reference image or video inputs for reference-to-video generation.
 * @param generateAudio - Whether the model should generate audio alongside the video.
 * @param providerOptions - Additional provider-specific options that are passed through to the provider
 * as body parameters.
 * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
 * @param abortSignal - An optional abort signal that can be used to cancel the call.
 * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
 *
 * @returns A result object that contains the generated videos.
 */
const defaultDownload = createDownload();

export async function experimental_generateVideo({
  model: modelArg,
  prompt: promptArg,
  n = 1,
  maxVideosPerCall,
  aspectRatio,
  resolution,
  duration,
  fps,
  seed,
  frameImages,
  inputReferences,
  generateAudio,
  providerOptions,
  maxRetries: maxRetriesArg,
  abortSignal,
  headers,
  download: downloadFn = defaultDownload,
}: {
  /**
   * The video model to use.
   */
  model: VideoModel;

  /**
   * The prompt that should be used to generate the video.
   */
  prompt: GenerateVideoPrompt;

  /**
   * Number of videos to generate.
   */
  n?: number;

  /**
   * Maximum number of videos per API call. If not provided, the model's default will be used.
   */
  maxVideosPerCall?: number;

  /**
   * Aspect ratio of the videos to generate. Must have the format `{width}:{height}`.
   */
  aspectRatio?: `${number}:${number}`;

  /**
   * Resolution of the videos to generate. Must have the format `{width}x{height}`.
   */
  resolution?: `${number}x${number}`;

  /**
   * Duration of the video in seconds.
   */
  duration?: number;

  /**
   * Frames per second for the video.
   */
  fps?: number;

  /**
   * Seed for the video generation.
   */
  seed?: number;

  /**
   * Role-tagged image inputs for image-to-video and first-last-frame generation.
   */
  frameImages?: Array<{
    /**
     * The image for this frame.
     */
    image: DataContent;

    /**
     * Which frame this image represents.
     */
    frameType: Experimental_VideoModelV4FrameType;
  }>;

  /**
   * Reference inputs for reference-to-video generation.
   *
   * Each entry may be a plain image/video ({@link DataContent}), or an object
   * form that carries an explicit `mediaType`.
   */
  inputReferences?: Array<
    | DataContent
    | {
        /**
         * The reference image or video.
         */
        data: DataContent;

        /**
         * The media type of the reference (e.g. 'image/png',
         * 'video/mp4').
         */
        mediaType?: string;
      }
  >;

  /**
   * Whether the model should generate audio alongside the video.
   */
  generateAudio?: boolean;

  /**
   * Additional provider-specific options that are passed through to the provider
   * as body parameters.
   */
  providerOptions?: ProviderOptions;

  /**
   * Maximum number of retries per video model call. Set to 0 to disable retries.
   *
   * @default 2
   */
  maxRetries?: number;

  /**
   * Abort signal.
   */
  abortSignal?: AbortSignal;

  /**
   * Additional headers to include in the request.
   * Only applicable for HTTP-based providers.
   */
  headers?: Record<string, string>;

  /**
   * Custom download function for fetching videos from URLs.
   * Use `createDownload()` from `ai` to create a download function with custom size limits.
   *
   * @default createDownload() (2 GiB limit)
   */
  download?: (options: {
    url: URL;
    abortSignal?: AbortSignal;
  }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>;
}): Promise<GenerateVideoResult> {
  const model = resolveVideoModel(modelArg);

  const headersWithUserAgent = withUserAgentSuffix(
    headers ?? {},
    `ai/${VERSION}`,
  );

  const { retry } = prepareRetries({
    maxRetries: maxRetriesArg,
    abortSignal,
  });

  const { prompt, image } = normalizePrompt(promptArg);

  const normalizedFrameImages:
    | Array<Experimental_VideoModelV4FrameImage>
    | undefined = frameImages?.flatMap(frame => {
    const normalizedImage = normalizeImageData(frame.image);
    return normalizedImage != null
      ? [{ image: normalizedImage, frameType: frame.frameType }]
      : [];
  });

  const normalizedInputReferences:
    | Array<Experimental_VideoModelV4File>
    | undefined = inputReferences?.flatMap(reference => {
    const normalized = normalizeReferenceData(reference);
    return normalized != null ? [normalized] : [];
  });

  const effectiveInputReferences =
    normalizedFrameImages != null && normalizedFrameImages.length > 0
      ? undefined
      : normalizedInputReferences;

  const warnings: Array<Warning> = [];

  if (
    normalizedFrameImages != null &&
    normalizedFrameImages.length > 0 &&
    normalizedInputReferences != null &&
    normalizedInputReferences.length > 0
  ) {
    warnings.push({
      type: 'other',
      message:
        'inputReferences were ignored because frameImages were provided; ' +
        'frameImages and inputReferences cannot be combined.',
    });
  }

  const firstFrameImage = normalizedFrameImages?.find(
    frame => frame.frameType === 'first_frame',
  )?.image;

  if (image != null && firstFrameImage != null) {
    warnings.push({
      type: 'other',
      message:
        'prompt.image was ignored because a first_frame frameImage was provided; ' +
        'the first_frame frameImage takes precedence as the start image.',
    });
  }

  const resolvedImage = firstFrameImage ?? image;

  const maxVideosPerCallWithDefault =
    maxVideosPerCall ?? (await invokeModelMaxVideosPerCall(model)) ?? 1;

  // parallelize calls to the model:
  const callCount = Math.ceil(n / maxVideosPerCallWithDefault);
  const callVideoCounts = Array.from({ length: callCount }, (_, index) => {
    const remaining = n - index * maxVideosPerCallWithDefault;
    return Math.min(remaining, maxVideosPerCallWithDefault);
  });

  const results = await Promise.all(
    callVideoCounts.map(
      async callVideoCount =>
        await retry(() =>
          model.doGenerate({
            prompt,
            n: callVideoCount,
            aspectRatio,
            resolution,
            duration,
            fps,
            seed,
            image: resolvedImage,
            frameImages: normalizedFrameImages,
            inputReferences: effectiveInputReferences,
            generateAudio,
            providerOptions: providerOptions ?? {},
            headers: headersWithUserAgent,
            abortSignal,
          } satisfies Experimental_VideoModelV4CallOptions),
        ),
    ),
  );

  // collect result videos, warnings, and response metadata
  const videos: Array<GeneratedFile> = [];
  const responses: Array<VideoModelResponseMetadata> = [];
  const providerMetadata: SharedV4ProviderMetadata = {};

  for (const result of results) {
    for (const videoData of result.videos) {
      switch (videoData.type) {
        case 'url': {
          const { data, mediaType: downloadedMediaType } = await downloadFn({
            url: new URL(videoData.url),
            abortSignal,
          });

          // Filter out generic/unknown media types that should fall through to detection
          const isUsableMediaType = (type: string | undefined): boolean =>
            !!type && type !== 'application/octet-stream';

          const mediaType =
            (isUsableMediaType(videoData.mediaType) && videoData.mediaType) ||
            (isUsableMediaType(downloadedMediaType) && downloadedMediaType) ||
            detectMediaType({
              data,
              topLevelType: 'video',
            }) ||
            'video/mp4';

          videos.push(
            new DefaultGeneratedFile({
              data,
              mediaType,
            }),
          );
          break;
        }

        case 'base64': {
          videos.push(
            new DefaultGeneratedFile({
              data: videoData.data,
              mediaType: videoData.mediaType || 'video/mp4',
            }),
          );
          break;
        }

        case 'binary': {
          const mediaType =
            videoData.mediaType ||
            detectMediaType({
              data: videoData.data,
              topLevelType: 'video',
            }) ||
            'video/mp4';

          videos.push(
            new DefaultGeneratedFile({
              data: videoData.data,
              mediaType,
            }),
          );
          break;
        }
      }
    }

    warnings.push(...result.warnings);

    responses.push({
      timestamp: result.response.timestamp,
      modelId: result.response.modelId,
      headers: result.response.headers,
      providerMetadata: result.providerMetadata,
    });

    if (result.providerMetadata != null) {
      for (const [providerName, metadata] of Object.entries(
        result.providerMetadata,
      )) {
        const existingMetadata = providerMetadata[providerName];
        if (existingMetadata != null && typeof existingMetadata === 'object') {
          providerMetadata[providerName] = {
            ...existingMetadata,
            ...metadata,
          };

          // Merge videos arrays if both exist
          if (
            'videos' in existingMetadata &&
            Array.isArray(existingMetadata.videos) &&
            'videos' in metadata &&
            Array.isArray(metadata.videos)
          ) {
            (providerMetadata[providerName] as { videos: unknown[] }).videos = [
              ...existingMetadata.videos,
              ...metadata.videos,
            ];
          }
        } else {
          providerMetadata[providerName] = metadata;
        }
      }
    }
  }

  if (videos.length === 0) {
    throw new NoVideoGeneratedError({ responses });
  }

  if (warnings.length > 0) {
    logWarnings({
      warnings,
      provider: model.provider,
      model: model.modelId,
    });
  }

  return {
    video: videos[0],
    videos,
    warnings,
    responses,
    providerMetadata,
  };
}

function normalizePrompt(promptArg: GenerateVideoPrompt): {
  prompt: string | undefined;
  image: Experimental_VideoModelV4File | undefined;
} {
  if (typeof promptArg === 'string') {
    return {
      prompt: promptArg,
      image: undefined,
    };
  }

  return {
    prompt: promptArg.text,
    image:
      promptArg.image != null ? normalizeImageData(promptArg.image) : undefined,
  };
}

function detectFileMediaType(
  data: Uint8Array,
  restrictToImages: boolean,
): string {
  const detected = restrictToImages
    ? detectMediaType({ data, topLevelType: 'image' })
    : detectMediaType({ data });
  return detected ?? 'image/png';
}

/**
 * Normalizes a {@link DataContent} image into a {@link Experimental_VideoModelV4File}.
 * Accepts a URL string, a data URL, a base64 string, or binary image data.
 */
function normalizeImageData(
  dataContent: DataContent,
  { restrictToImages = true }: { restrictToImages?: boolean } = {},
): Experimental_VideoModelV4File | undefined {
  if (typeof dataContent === 'string') {
    if (
      dataContent.startsWith('http://') ||
      dataContent.startsWith('https://')
    ) {
      return {
        type: 'url',
        url: dataContent,
      };
    }

    if (dataContent.startsWith('data:')) {
      const { mediaType, base64Content } = splitDataUrl(dataContent);
      const data = convertBase64ToUint8Array(base64Content ?? '');
      return {
        type: 'file',
        mediaType: mediaType ?? detectFileMediaType(data, restrictToImages),
        data,
      };
    }

    const bytes = convertBase64ToUint8Array(dataContent);
    return {
      type: 'file',
      mediaType: detectFileMediaType(bytes, restrictToImages),
      data: bytes,
    };
  }

  if (dataContent instanceof Uint8Array || dataContent instanceof ArrayBuffer) {
    const bytes =
      dataContent instanceof Uint8Array
        ? dataContent
        : new Uint8Array(dataContent);
    return {
      type: 'file',
      mediaType: detectFileMediaType(bytes, restrictToImages),
      data: bytes,
    };
  }

  return undefined;
}

/**
 * Normalizes a reference input into a {@link Experimental_VideoModelV4File},
 * accepting either a plain {@link DataContent} or the object form that carries
 * an explicit `mediaType`.
 */
function normalizeReferenceData(
  reference:
    | DataContent
    | {
        data: DataContent;
        mediaType?: string;
      },
): Experimental_VideoModelV4File | undefined {
  const isObjectForm =
    typeof reference === 'object' &&
    reference != null &&
    !(reference instanceof Uint8Array) &&
    !(reference instanceof ArrayBuffer) &&
    'data' in reference;

  if (!isObjectForm) {
    return normalizeImageData(reference as DataContent, {
      restrictToImages: false,
    });
  }

  const normalized = normalizeImageData(reference.data, {
    restrictToImages: false,
  });
  if (normalized == null) {
    return normalized;
  }

  return {
    ...normalized,
    ...(reference.mediaType != null ? { mediaType: reference.mediaType } : {}),
  };
}

async function invokeModelMaxVideosPerCall(model: Experimental_VideoModelV4) {
  if (typeof model.maxVideosPerCall === 'function') {
    return await model.maxVideosPerCall({ modelId: model.modelId });
  }

  return model.maxVideosPerCall;
}