@omnimedia/omnitool/s/driver/fns/schematic.ts

open source video processing tools

import {TextStyleOptions} from "pixi.js"
import {AsSchematic} from "@e280/comrade"
import type {AudioEncodingConfig, StreamTargetChunk, VideoEncodingConfig} from "mediabunny"

import {Id} from "../../timeline/index.js"
import {Crop} from "../../timeline/parts/item.js"
import {Mat6} from "../../timeline/utils/matrix.js"
import {FilterParams, FilterType} from "../../timeline/parts/filters.js"
import {TransitionName} from "../../timeline/parts/transitions.js"

export type DriverSchematic = AsSchematic<{

	// happens on the web worker
	work: {
		hello(): Promise<void>

		decodeAudio(input: {
			source: DecoderSource
			cancel: MessagePort
			audio: WritableStream<AudioData>
			start?: number
			end?: number
		}): Promise<void>

		decodeVideo(input: {
			source: DecoderSource
			cancel: MessagePort
			video: WritableStream<VideoFrame>
			start?: number
			end?: number
		}): Promise<void>

		encode(input: EncoderInput & {writable: WritableStream<StreamTargetChunk>}): Promise<void>
	}

	// happens on the main thread
	host: {
		world(): Promise<void>
	}
}>

export interface EncoderInput {
	video?: ReadableStream<VideoFrame>
	audio?: ReadableStream<AudioData>
	config: RenderConfig
}

export interface RenderConfig {
	video: VideoEncodingConfig
	audio: AudioEncodingConfig
}

export type DecoderSource = Blob | string | URL

export interface DecoderInput {
	source: DecoderSource
	start?: number
	end?: number
	onFrame?: (frame: VideoFrame) => Promise<VideoFrame>
}

export interface MuxOpts {
	config: {
		video: {
			width: number
			height: number
		},
		audio?: {
			codec: "opus" | "aac"
			numberOfChannels: number
			sampleRate: number
		}
	}
}

export type Composition = Layer | (Layer | Composition)[]

export type FilterSpec = {
	[TFilter in FilterType]: {
		type: TFilter
		params?: FilterParams<TFilter>
	}
}[FilterType]

export type TextLayer = {
	id: Id
	kind: 'text'
	content: string
	style?: TextStyleOptions
	matrix?: Mat6
	alpha?: number
	crop?: Crop
	filters?: FilterSpec[]
}

export type ImageLayer = {
	id: Id
	kind: 'image'
	frame: VideoFrame
	matrix?: Mat6
	alpha?: number
	crop?: Crop
	filters?: FilterSpec[]
}

export type TransitionLayer = {
	id: Id
	kind: 'transition'
	name: TransitionName
	progress: number
	from: ImageLayer
	to: ImageLayer
}

export type GapLayer = {
	id: Id
	kind: 'gap'
}

export type Audio = {
	id: Id
	kind: "audio"
	data: AudioData
}

export type Layer = TextLayer | ImageLayer | TransitionLayer | GapLayer