@omnimedia/omnitool
Version:
open source video processing tools
306 lines • 11.8 kB
JavaScript
import { pub } from "@e280/stz";
import { autoDetectRenderer, ColorMatrixFilter, Container, Graphics, RenderTexture, Sprite, Text, Texture } from "pixi.js";
import * as PixiFilters from "pixi-filters";
import { findPixiFilter } from "../utils/find-pixi-filter.js";
import { mat6ToMatrix } from "../../timeline/utils/matrix.js";
import { makeTransition } from "../../features/transition/transition.js";
export class Compositor {
pixi;
onPointerDown = pub();
onPointerMove = pub();
onPointerUp = pub();
onDispose = pub();
static async setup() {
const renderer = await autoDetectRenderer({
width: 1920,
height: 1080,
preference: "webgl", // webgl and webgl2 causes memory leaks on chrome
background: "black",
preferWebGLVersion: 2
});
const stage = new Container();
stage.interactive = true;
return new this({ renderer, stage });
}
constructor(pixi) {
this.pixi = pixi;
}
#transitions = new Map();
// objects rendered for current Composition
#activeObjects = new Map();
#cropMasks = new WeakMap();
resize(width, height) {
this.pixi.renderer.resize(width, height);
}
async composite(composition) {
const { stage, renderer } = this.pixi;
const activeIds = this.#collectIds(composition);
this.#cleanup(activeIds);
this.#cleanupTransitions(activeIds);
const { dispose } = await this.#renderLayer(composition, stage);
renderer.render(stage);
// make sure browser support webgl/webgpu otherwise it might take much longer to construct frame
// if its very slow on eg edge try chrome
const frame = new VideoFrame(renderer.canvas, {
timestamp: 0,
duration: 0,
});
dispose();
return frame;
}
/**
* get object for current Composition
* */
getActiveObject(id) {
return this.#activeObjects.get(id)?.sprite;
}
async #renderLayer(layer, parent) {
if (Array.isArray(layer)) {
const disposers = [];
for (const child of [...layer].reverse()) {
const result = await this.#renderLayer(child, parent);
disposers.push(result.dispose);
}
return { dispose: () => disposers.forEach(d => d()) };
}
switch (layer.kind) {
case 'text':
return this.#renderTextLayer(layer, parent);
case 'image':
return this.#renderImageLayer(layer, parent);
case 'transition':
return this.#renderTransitionLayer(layer, parent);
case 'gap': {
this.pixi?.renderer.clear();
return { dispose: () => { } };
}
default:
console.warn('Unknown layer kind', layer.kind);
return { dispose: () => { } };
}
}
#renderTextLayer(layer, parent) {
const sprite = this.#findOrCreate(layer);
sprite.text = layer.content;
if (layer.style)
sprite.style = layer.style;
this.#applyTransform(sprite, layer.matrix);
this.#applyAlpha(sprite, layer.alpha);
this.#applyCrop(sprite, layer.crop);
this.#applyFilters(sprite, layer.filters);
parent.addChild(sprite);
return {
dispose: () => { }
};
}
#renderImageLayer(layer, parent) {
const sprite = this.#findOrCreate(layer);
if (sprite.texture && sprite.texture !== Texture.EMPTY) {
sprite.texture.destroy(true);
}
this.#applyImageLayer(sprite, layer);
parent.addChild(sprite);
return {
dispose: () => {
layer.frame.close();
}
};
}
#renderTransitionLayer({ id, from, to, progress, name }, parent) {
const transition = this.#transitions.get(id) ?? this.#createTransition(id, name);
const { renderer } = this.pixi;
transition.from.resize(renderer.width, renderer.height);
transition.to.resize(renderer.width, renderer.height);
this.#renderImageToTexture(from, transition.from);
this.#renderImageToTexture(to, transition.to);
const texture = transition.transition.render({
from: transition.from,
to: transition.to,
progress,
width: renderer.width,
height: renderer.height
});
const sprite = new Sprite(texture);
parent.addChild(sprite);
return { dispose: () => sprite.destroy(false) };
}
#createTransition(id, name) {
const transition = {
transition: makeTransition({ name, renderer: this.pixi.renderer }),
from: RenderTexture.create({ width: 1, height: 1 }),
to: RenderTexture.create({ width: 1, height: 1 })
};
this.#transitions.set(id, transition);
return transition;
}
#renderImageToTexture(layer, target) {
const sprite = new Sprite();
const texture = this.#applyImageLayer(sprite, layer);
this.pixi.renderer.render({ container: sprite, target, clear: true });
sprite.destroy({ children: true });
texture.destroy(true);
layer.frame.close();
}
#applyImageLayer(sprite, layer) {
const texture = Texture.from(layer.frame);
sprite.texture = texture;
this.#applyTransform(sprite, layer.matrix);
this.#applyAlpha(sprite, layer.alpha);
this.#applyCrop(sprite, layer.crop);
this.#applyFilters(sprite, layer.filters);
return texture;
}
#applyTransform(target, worldMatrix) {
if (!worldMatrix)
return;
const mx = mat6ToMatrix(worldMatrix);
target.setFromMatrix(mx);
}
#applyAlpha(target, alpha) {
target.alpha = alpha ?? 1;
}
#applyCrop(target, crop) {
const existing = this.#cropMasks.get(target);
if (existing) {
if (target.mask === existing)
target.mask = null;
existing.removeFromParent();
existing.clear();
}
if (!crop || crop.every(value => value === 0))
return;
const [top, right, bottom, left] = crop;
const activeMask = target.mask;
target.mask = null;
const bounds = target.getLocalBounds();
target.mask = activeMask;
const x = bounds.x + bounds.width * left;
const y = bounds.y + bounds.height * top;
const width = bounds.width * Math.max(0, 1 - left - right);
const height = bounds.height * Math.max(0, 1 - top - bottom);
const mask = existing ?? new Graphics();
mask.clear();
mask.beginFill(0xffffff);
mask.drawRect(x, y, width, height);
mask.endFill();
target.addChild(mask);
target.mask = mask;
this.#cropMasks.set(target, mask);
}
#applyFilters(target, specs) {
if (!specs?.length) {
target.filters = null;
return;
}
target.filters = specs
.map((spec) => {
switch (spec.type) {
case "ColorMatrixFilter": {
const { matrix, ...params } = spec.params ?? {};
const filter = new ColorMatrixFilter(params);
if (matrix)
filter.matrix = matrix;
return filter;
}
case "EmbossFilter":
return new PixiFilters.EmbossFilter(spec.params?.strength);
case "PixelateFilter":
return new PixiFilters.PixelateFilter(spec.params?.size);
default: {
const PixiFilter = findPixiFilter(spec.type);
return PixiFilter ? new PixiFilter(spec.params) : undefined;
}
}
})
.filter((filter) => !!filter);
}
#findOrCreate(layer) {
const object = this.#activeObjects.get(layer.id);
if (!object) {
switch (layer.kind) {
case 'text': {
const text = new Text({
text: layer.content,
style: layer.style
});
text.eventMode = "static";
const down = (event) => this.onPointerDown.publish({ event, id: layer.id, object: text });
const up = (event) => this.onPointerUp.publish({ event, id: layer.id, object: text });
const move = (event) => this.onPointerUp.publish({ event, id: layer.id, object: text });
text.on("pointerdown", down);
text.on("pointerup", up);
text.on("pointermove", move);
return this.#activeObjects
.set(layer.id, {
sprite: text,
dispose: () => {
this.onDispose.publish({ id: layer.id, object: text });
text.off("pointermove", move);
text.off("pointerdown", down);
text.off("pointerup", up);
}
})
.get(layer.id)?.sprite;
}
case 'image': {
const sprite = new Sprite();
sprite.eventMode = "static";
const down = (event) => this.onPointerDown.publish({ event, id: layer.id, object: sprite });
const up = (event) => this.onPointerUp.publish({ event, id: layer.id, object: sprite });
const move = (event) => this.onPointerUp.publish({ event, id: layer.id, object: sprite });
sprite.on("pointerdown", down);
sprite.on("pointermove", move);
sprite.on("pointerup", up);
return this.#activeObjects
.set(layer.id, {
sprite,
dispose: () => {
this.onDispose.publish({ id: layer.id, object: sprite });
sprite.off("pointerdown", down);
sprite.off("pointermove", move);
sprite.off("pointerup", up);
}
})
.get(layer.id)?.sprite;
}
}
}
else
return object.sprite;
}
#collectIds(layers) {
const result = new Set();
const traverse = (node) => {
if (Array.isArray(node)) {
for (const child of node)
traverse(child);
}
else {
result.add(node.id);
}
};
traverse(layers);
return result;
}
#cleanup(activeIds) {
for (const id of this.#activeObjects.keys()) {
if (!activeIds.has(id)) {
const { sprite, dispose } = this.#activeObjects.get(id);
dispose();
sprite.destroy(true);
this.#activeObjects.delete(id);
}
}
}
#cleanupTransitions(activeIds) {
for (const [id, cached] of this.#transitions) {
if (!activeIds.has(id)) {
cached.transition.dispose();
cached.from.destroy(true);
cached.to.destroy(true);
this.#transitions.delete(id);
}
}
}
}
//# sourceMappingURL=compositor.js.map