@tanstack/ai/src/activities/generateVideo/adapter.ts

Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.

import { arrayBufferToBase64 } from '@tanstack/ai-utils'
import { snapToDurationOption } from './snap'
import type {
  ModelInputModalitiesByName,
  VideoGenerationOptions,
  VideoJobResult,
  VideoStatusResult,
  VideoStreamResult,
  VideoUrlResult,
} from '../../types'

/**
 * Structured description of the durations a video model accepts.
 *
 * Tagged union so the same shape can express discrete enums (OpenAI Sora,
 * Veo), continuous ranges, mixed shapes, and models with no duration field.
 * Consumed by `VideoAdapter.availableDurations()`.
 *
 * @experimental Video generation is an experimental feature and may change.
 */
export type DurationOptions<T extends string | number | undefined> =
  | { kind: 'discrete'; values: ReadonlyArray<NonNullable<T>> }
  | { kind: 'range'; min: number; max: number; step?: number; unit: 'seconds' }
  | {
      kind: 'mixed'
      values: ReadonlyArray<NonNullable<T>>
      range?: { min: number; max: number; step?: number }
    }
  | { kind: 'none' }

/**
 * Spellings of one clip length: the number `6`, the string `"6"`, or the
 * template `"6s"`.
 */
export type VideoDurationSpell<N extends number> = N | `${N}` | `${N}s`

/**
 * Configuration for video adapter instances
 *
 * @experimental Video generation is an experimental feature and may change.
 */
export interface VideoAdapterConfig {
  apiKey?: string
  baseUrl?: string
  timeout?: number
  maxRetries?: number
  headers?: Record<string, string>
}

/**
 * Video adapter interface with pre-resolved generics.
 *
 * An adapter is created by a provider function: `provider('model')` → `adapter`
 * All type resolution happens at the provider call site, not in this interface.
 *
 * @experimental Video generation is an experimental feature and may change.
 *
 * Generic parameters:
 * - TModel: The specific model name (e.g., 'sora-2')
 * - TProviderOptions: Provider-specific options (already resolved)
 * - TModelProviderOptionsByName: Map from model name to its specific provider options
 * - TModelSizeByName: Map from model name to its supported sizes
 * - TModelInputModalitiesByName: Map from model name to the non-text prompt
 *   modalities it accepts (constrains the `prompt` part types at compile time)
 * - TModelDurationByName: Map from model name to its supported duration
 *   union. Defaults to `Record<string, number>` so adapters that haven't
 *   declared a map keep today's `duration?: number` typing.
 */
export interface VideoAdapter<
  TModel extends string = string,
  TProviderOptions extends object = Record<string, unknown>,
  TModelProviderOptionsByName extends Record<string, any> = Record<string, any>,
  TModelSizeByName extends Record<string, string | undefined> = Record<
    string,
    string
  >,
  TModelInputModalitiesByName extends ModelInputModalitiesByName =
    ModelInputModalitiesByName,
  TModelDurationByName extends Record<string, string | number | undefined> =
    Record<string, number>,
> {
  /** Discriminator for adapter kind - used to determine API shape */
  readonly kind: 'video'
  /** Adapter name identifier */
  readonly name: string
  /**
   * Declares that this adapter can consume `{ type: 'file' }` content
   * sources (provider Files API references). The activity dispatcher rejects
   * file sources in preflight for adapters that don't declare this, so
   * adapters written before the file arm existed fail closed.
   */
  readonly supportsFileSources?: boolean
  /** The model this adapter is configured for */
  readonly model: TModel

  /**
   * @internal Type-only properties for inference. Not assigned at runtime.
   */
  '~types': {
    providerOptions: TProviderOptions
    modelProviderOptionsByName: TModelProviderOptionsByName
    modelSizeByName: TModelSizeByName
    modelInputModalitiesByName: TModelInputModalitiesByName
    modelDurationByName: TModelDurationByName
  }

  /**
   * Create a new video generation job.
   * Returns a job ID that can be used to poll for status and retrieve the video.
   */
  createVideoJob: (
    options: VideoGenerationOptions<
      TProviderOptions,
      TModelSizeByName[TModel],
      TModelDurationByName[TModel]
    >,
  ) => Promise<VideoJobResult>

  /**
   * Get the current status of a video generation job.
   */
  getVideoStatus: (jobId: string) => Promise<VideoStatusResult>

  /**
   * Get the finished video: a public URL when the provider has one, or the
   * download stream for generation middleware to host. Call only after
   * status is 'completed'.
   *
   * Optional only so adapters written against `getVideoUrl` keep working.
   * New adapters implement this.
   */
  getVideo?: (jobId: string) => Promise<VideoUrlResult | VideoStreamResult>

  /**
   * @deprecated Use `getVideo`. This is `getVideo` with a provider stream
   * buffered into a base64 `data:` URL, which holds the whole video in memory.
   */
  getVideoUrl: (jobId: string) => Promise<VideoUrlResult>

  /**
   * Describe the durations this adapter's model accepts. Returns a tagged
   * union so consumers can render UI / coerce input without provider-specific
   * knowledge.
   */
  availableDurations: () => DurationOptions<TModelDurationByName[TModel]>

  /**
   * Coerce `input` to the closest duration this model accepts.
   * `input` may be seconds (`7`), a numeric string (`"7"`), a template
   * (`"6s"`), or a keyword the model lists (`"auto"`).
   * Returns `undefined` when the model has no duration field, or when
   * `input` is a keyword that model does not list.
   */
  snapDuration: (
    input: number | string,
  ) => TModelDurationByName[TModel] | undefined
}

const LARGE_VIDEO_BYTES = 10 * 1024 * 1024

/**
 * The fallback when nothing hosts a provider video stream: buffer it into a
 * base64 `data:` URL. A result that already has a `url` passes through.
 */
export async function inlineVideoStream<
  T extends {
    url?: string
    body?: ReadableStream<Uint8Array>
    contentType?: string
  },
>(video: T): Promise<Omit<T, 'body' | 'contentType'> & { url: string }> {
  const { body, contentType, ...rest } = video
  if (rest.url !== undefined) return { ...rest, url: rest.url }
  if (!body) throw new Error('Video result has no url and no body')
  const buffer = await new Response(body).arrayBuffer()
  if (buffer.byteLength > LARGE_VIDEO_BYTES) {
    console.warn(
      `[generateVideo] buffered ${(buffer.byteLength / 1024 / 1024).toFixed(1)} MiB of video into memory for a base64 data: URL. ` +
        `Workers/serverless runtimes commonly run out of memory above ~10 MiB. ` +
        `Add withGenerationPersistence with artifactUrl to stream it into your storage instead.`,
    )
  }
  return {
    ...rest,
    url: `data:${contentType ?? 'video/mp4'};base64,${arrayBufferToBase64(buffer)}`,
  }
}

/**
 * A VideoAdapter with any/unknown type parameters.
 * Useful as a constraint in generic functions and interfaces.
 */
export type AnyVideoAdapter = VideoAdapter<any, any, any, any, any, any>

/**
 * Abstract base class for video generation adapters.
 * Extend this class to implement a video adapter for a specific provider.
 *
 * @experimental Video generation is an experimental feature and may change.
 *
 * Generic parameters match VideoAdapter - all pre-resolved by the provider function.
 */
export abstract class BaseVideoAdapter<
  TModel extends string = string,
  TProviderOptions extends object = Record<string, unknown>,
  TModelProviderOptionsByName extends Record<string, any> = Record<string, any>,
  TModelSizeByName extends Record<string, string | undefined> = Record<
    string,
    string
  >,
  TModelInputModalitiesByName extends ModelInputModalitiesByName =
    ModelInputModalitiesByName,
  TModelDurationByName extends Record<string, string | number | undefined> =
    Record<string, number>,
> implements VideoAdapter<
  TModel,
  TProviderOptions,
  TModelProviderOptionsByName,
  TModelSizeByName,
  TModelInputModalitiesByName,
  TModelDurationByName
> {
  readonly kind = 'video' as const
  abstract readonly name: string
  readonly supportsFileSources: boolean = false
  readonly model: TModel

  // Type-only property - never assigned at runtime
  declare '~types': {
    providerOptions: TProviderOptions
    modelProviderOptionsByName: TModelProviderOptionsByName
    modelSizeByName: TModelSizeByName
    modelInputModalitiesByName: TModelInputModalitiesByName
    modelDurationByName: TModelDurationByName
  }

  protected config: VideoAdapterConfig

  constructor(config: VideoAdapterConfig = {}, model: TModel) {
    this.config = config
    this.model = model
  }

  abstract createVideoJob(
    options: VideoGenerationOptions<
      TProviderOptions,
      TModelSizeByName[TModel],
      TModelDurationByName[TModel]
    >,
  ): Promise<VideoJobResult>

  abstract getVideoStatus(jobId: string): Promise<VideoStatusResult>

  /** Implement this. See {@link VideoAdapter.getVideo}. */
  getVideo?(jobId: string): Promise<VideoUrlResult | VideoStreamResult>

  /**
   * @deprecated Implement and call `getVideo`. This is `getVideo` with a
   * provider stream buffered into a base64 `data:` URL.
   */
  async getVideoUrl(jobId: string): Promise<VideoUrlResult> {
    if (!this.getVideo) {
      throw new Error(`${this.name}: video adapter must implement getVideo()`)
    }
    return await inlineVideoStream(await this.getVideo(jobId))
  }

  /**
   * Default implementation returns `{ kind: 'none' }`. Adapters that have
   * declared their per-model duration map should override this.
   */
  availableDurations(): DurationOptions<TModelDurationByName[TModel]> {
    return { kind: 'none' }
  }

  /**
   * Uses `availableDurations()`. Adapters that declare a duration map only
   * need to override that method.
   */
  snapDuration(
    input: number | string,
  ): TModelDurationByName[TModel] | undefined {
    return snapToDurationOption(input, this.availableDurations())
  }

  protected generateId(): string {
    return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
  }
}