mediabunny
Version:
Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.
1,074 lines • 102 kB
JavaScript
/*!
* Copyright (c) 2026-present, Vanilagy and contributors
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
if (value !== null && value !== void 0) {
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
var dispose, inner;
if (async) {
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
dispose = value[Symbol.asyncDispose];
}
if (dispose === void 0) {
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
dispose = value[Symbol.dispose];
if (async) inner = dispose;
}
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
env.stack.push({ value: value, dispose: dispose, async: async });
}
else if (async) {
env.stack.push({ async: true });
}
return value;
};
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
return function (env) {
function fail(e) {
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
env.hasError = true;
}
var r, s = 0;
function next() {
while (r = env.stack.pop()) {
try {
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
if (r.dispose) {
var result = r.dispose.call(r.value);
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
}
else s |= 1;
}
catch (e) {
fail(e);
}
}
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
if (env.hasError) throw env.error;
}
return next();
};
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
var e = new Error(message);
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
});
import { assert, clamp, COLOR_PRIMARIES_MAP, isAllowSharedBufferSource, MATRIX_COEFFICIENTS_MAP, SECOND_TO_MICROSECOND_FACTOR, toDataView, toUint8Array, TRANSFER_CHARACTERISTICS_MAP, isFirefox, polyfillSymbolDispose, assertNever, isWebKit, simplifyRational, validateRectangle, normalizeRotation, roundToMultiple, arrayArgmin, } from './misc.js';
polyfillSymbolDispose();
// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they
// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught.
let lastVideoGcErrorLog = -Infinity;
let lastAudioGcErrorLog = -Infinity;
let finalizationRegistry = null;
if (typeof FinalizationRegistry !== 'undefined') {
finalizationRegistry = new FinalizationRegistry((value) => {
const now = performance.now();
if (value.type === 'video') {
if (now - lastVideoGcErrorLog >= 1000) {
// This error is annoying but oh so important
console.error(`A VideoSample was garbage collected without first being closed. For proper resource management,`
+ ` make sure to call close() on all your VideoSamples as soon as you're done using them.`);
lastVideoGcErrorLog = now;
}
if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) {
value.data.close(); // Prevent the browser error since we're logging our own
}
}
else {
if (now - lastAudioGcErrorLog >= 1000) {
console.error(`An AudioSample was garbage collected without first being closed. For proper resource management,`
+ ` make sure to call close() on all your AudioSamples as soon as you're done using them.`);
lastAudioGcErrorLog = now;
}
if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) {
value.data.close();
}
}
});
}
/**
* Abstract base class for custom video sample resources. Implement this class to provide custom backing
* for VideoSample instances.
* @group Samples
* @public
*/
export class VideoSampleResource {
constructor() {
/** @internal */
this._referenceCount = 0;
/** @internal */
this._lastAllocationBuffer = null;
}
}
/**
* The list of {@link VideoSample} pixel formats.
* @group Samples
* @public
*/
export const VIDEO_SAMPLE_PIXEL_FORMATS = [
// 4:2:0 Y, U, V
'I420',
'I420P10',
'I420P12',
// 4:2:0 Y, U, V, A
'I420A',
'I420AP10',
'I420AP12',
// 4:2:2 Y, U, V
'I422',
'I422P10',
'I422P12',
// 4:2:2 Y, U, V, A
'I422A',
'I422AP10',
'I422AP12',
// 4:4:4 Y, U, V
'I444',
'I444P10',
'I444P12',
// 4:4:4 Y, U, V, A
'I444A',
'I444AP10',
'I444AP12',
// 4:2:0 Y, UV
'NV12',
// 4:4:4 RGBA
'RGBA',
// 4:4:4 RGBX (opaque)
'RGBX',
// 4:4:4 BGRA
'BGRA',
// 4:4:4 BGRX (opaque)
'BGRX',
];
const VIDEO_SAMPLE_PIXEL_FORMATS_SET = new Set(VIDEO_SAMPLE_PIXEL_FORMATS);
/**
* Represents a raw, unencoded video sample (frame). Mainly used as an expressive wrapper around WebCodecs API's
* [`VideoFrame`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame), but can also be used standalone.
* @group Samples
* @public
*/
export class VideoSample {
/** The width of the frame in pixels. */
get codedWidth() {
// This is wrong, but the fix is a v2 thing
return this.visibleRect.width;
}
/** The height of the frame in pixels. */
get codedHeight() {
// Same here
return this.visibleRect.height;
}
/** The display width of the frame in pixels, after aspect ratio adjustment and rotation. */
get displayWidth() {
return this.rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight;
}
/** The display height of the frame in pixels, after aspect ratio adjustment and rotation. */
get displayHeight() {
return this.rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth;
}
/** The presentation timestamp of the frame in microseconds. */
get microsecondTimestamp() {
return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.timestamp);
}
/** The duration of the frame in microseconds. */
get microsecondDuration() {
return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration);
}
/**
* Whether this sample uses a pixel format that can hold transparency data. Note that this doesn't necessarily mean
* that the sample is transparent.
*/
get hasAlpha() {
return this.format && this.format.includes('A');
}
constructor(data, init) {
/** @internal */
this._closed = false;
if (data instanceof ArrayBuffer
|| (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer)
|| ArrayBuffer.isView(data)) {
if (!init || typeof init !== 'object') {
throw new TypeError('init must be an object.');
}
if (init.format === undefined || !VIDEO_SAMPLE_PIXEL_FORMATS_SET.has(init.format)) {
throw new TypeError('init.format must be one of: ' + VIDEO_SAMPLE_PIXEL_FORMATS.join(', '));
}
if (!Number.isInteger(init.codedWidth) || init.codedWidth <= 0) {
throw new TypeError('init.codedWidth must be a positive integer.');
}
if (!Number.isInteger(init.codedHeight) || init.codedHeight <= 0) {
throw new TypeError('init.codedHeight must be a positive integer.');
}
if (init.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
}
if (!Number.isFinite(init.timestamp)) {
throw new TypeError('init.timestamp must be a number.');
}
if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
throw new TypeError('init.duration, when provided, must be a non-negative number.');
}
if (init.layout !== undefined) {
if (!Array.isArray(init.layout)) {
throw new TypeError('init.layout, when provided, must be an array.');
}
for (const plane of init.layout) {
if (!plane || typeof plane !== 'object' || Array.isArray(plane)) {
throw new TypeError('Each entry in init.layout must be an object.');
}
if (!Number.isInteger(plane.offset) || plane.offset < 0) {
throw new TypeError('plane.offset must be a non-negative integer.');
}
if (!Number.isInteger(plane.stride) || plane.stride < 0) {
throw new TypeError('plane.stride must be a non-negative integer.');
}
}
}
if (init.visibleRect !== undefined) {
validateRectangle(init.visibleRect, 'init.visibleRect');
}
if (init.displayWidth !== undefined
&& (!Number.isInteger(init.displayWidth) || init.displayWidth <= 0)) {
throw new TypeError('init.displayWidth, when provided, must be a positive integer.');
}
if (init.displayHeight !== undefined
&& (!Number.isInteger(init.displayHeight) || init.displayHeight <= 0)) {
throw new TypeError('init.displayHeight, when provided, must be a positive integer.');
}
if ((init.displayWidth !== undefined) !== (init.displayHeight !== undefined)) {
throw new TypeError('init.displayWidth and init.displayHeight must be either both provided or both omitted.');
}
this._data = toUint8Array(data).slice(); // Copy it
this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth, init.codedHeight);
this.format = init.format;
this.rotation = init.rotation ?? 0;
this.timestamp = init.timestamp;
this.duration = init.duration ?? 0;
let colorSpaceInit = init.colorSpace ?? null;
if (colorSpaceInit === null) {
if (this.format === 'RGBA' || this.format === 'RGBX'
|| this.format === 'BGRA' || this.format === 'BGRX') {
// sRGB Color Space
colorSpaceInit = {
primaries: 'bt709',
transfer: 'iec61966-2-1',
matrix: 'rgb',
fullRange: true,
};
}
else {
// REC709 Color Space
colorSpaceInit = {
primaries: 'bt709',
transfer: 'bt709',
matrix: 'bt709',
fullRange: false,
};
}
}
this.colorSpace = new VideoSampleColorSpace(colorSpaceInit);
this.visibleRect = {
left: init.visibleRect?.left ?? 0,
top: init.visibleRect?.top ?? 0,
width: init.visibleRect?.width ?? init.codedWidth,
height: init.visibleRect?.height ?? init.codedHeight,
};
if (init.displayWidth !== undefined) {
this.squarePixelWidth = this.rotation % 180 === 0 ? init.displayWidth : init.displayHeight;
this.squarePixelHeight = this.rotation % 180 === 0 ? init.displayHeight : init.displayWidth;
}
else {
this.squarePixelWidth = this.visibleRect.width;
this.squarePixelHeight = this.visibleRect.height;
}
}
else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) {
if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
}
if (init?.timestamp !== undefined && !Number.isFinite(init?.timestamp)) {
throw new TypeError('init.timestamp, when provided, must be a number.');
}
if (init?.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
throw new TypeError('init.duration, when provided, must be a non-negative number.');
}
if (init?.visibleRect !== undefined) {
validateRectangle(init.visibleRect, 'init.visibleRect');
}
this._data = data;
this._layout = null;
this.format = data.format;
this.visibleRect = {
left: data.visibleRect?.x ?? 0,
top: data.visibleRect?.y ?? 0,
width: data.visibleRect?.width ?? data.codedWidth,
height: data.visibleRect?.height ?? data.codedHeight,
};
// The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application
// where the browser makes use of it. If a case gets found, I'll add it.
this.rotation = init?.rotation ?? 0;
// Assuming no innate VideoFrame rotation here
this.squarePixelWidth = data.displayWidth;
this.squarePixelHeight = data.displayHeight;
this.timestamp = init?.timestamp ?? data.timestamp / 1e6;
this.duration = init?.duration ?? (data.duration ?? 0) / 1e6;
this.colorSpace = new VideoSampleColorSpace(data.colorSpace);
}
else if ((typeof HTMLImageElement !== 'undefined' && data instanceof HTMLImageElement)
|| (typeof SVGImageElement !== 'undefined' && data instanceof SVGImageElement)
|| (typeof ImageBitmap !== 'undefined' && data instanceof ImageBitmap)
|| (typeof HTMLVideoElement !== 'undefined' && data instanceof HTMLVideoElement)
|| (typeof HTMLCanvasElement !== 'undefined' && data instanceof HTMLCanvasElement)
|| (typeof OffscreenCanvas !== 'undefined' && data instanceof OffscreenCanvas)) {
if (!init || typeof init !== 'object') {
throw new TypeError('init must be an object.');
}
if (init.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
}
if (!Number.isFinite(init.timestamp)) {
throw new TypeError('init.timestamp must be a number.');
}
if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
throw new TypeError('init.duration, when provided, must be a non-negative number.');
}
if (typeof VideoFrame !== 'undefined') {
return new VideoSample(new VideoFrame(data, {
timestamp: Math.trunc(init.timestamp * SECOND_TO_MICROSECOND_FACTOR),
// Drag 0 to undefined
duration: Math.trunc((init.duration ?? 0) * SECOND_TO_MICROSECOND_FACTOR) || undefined,
}), init);
}
let width = 0;
let height = 0;
// Determine the dimensions of the thing
if ('naturalWidth' in data) {
width = data.naturalWidth;
height = data.naturalHeight;
}
else if ('videoWidth' in data) {
width = data.videoWidth;
height = data.videoHeight;
}
else if ('width' in data) {
width = Number(data.width);
height = Number(data.height);
}
if (!width || !height) {
throw new TypeError('Could not determine dimensions.');
}
const canvas = new OffscreenCanvas(width, height);
const context = canvas.getContext('2d', {
alpha: isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
willReadFrequently: true,
});
assert(context);
// Draw it to a canvas
context.drawImage(data, 0, 0);
this._data = canvas;
this._layout = null;
this.format = 'RGBX';
this.visibleRect = { left: 0, top: 0, width, height };
this.squarePixelWidth = width;
this.squarePixelHeight = height;
this.rotation = init.rotation ?? 0;
this.timestamp = init.timestamp;
this.duration = init.duration ?? 0;
this.colorSpace = new VideoSampleColorSpace({
matrix: 'rgb',
primaries: 'bt709',
transfer: 'iec61966-2-1',
fullRange: true,
});
}
else if (data instanceof VideoSampleResource) {
if (!init || typeof init !== 'object') {
throw new TypeError('init must be an object.');
}
if (init.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) {
throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.');
}
if (!Number.isFinite(init.timestamp)) {
throw new TypeError('init.timestamp must be a number.');
}
if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) {
throw new TypeError('init.duration, when provided, must be a non-negative number.');
}
this._data = data;
data._referenceCount++;
this.format = data.getFormat();
if (this.format !== null && !VIDEO_SAMPLE_PIXEL_FORMATS.includes(this.format)) {
throw new TypeError('getFormat() must return a VideoSamplePixelFormat or null.');
}
this.visibleRect = {
left: 0,
top: 0,
width: data.getCodedWidth(),
height: data.getCodedHeight(),
};
if (!Number.isInteger(this.visibleRect.width) || this.visibleRect.width <= 0) {
throw new TypeError('getCodedWidth() must return a positive integer.');
}
if (!Number.isInteger(this.visibleRect.height) || this.visibleRect.height <= 0) {
throw new TypeError('getCodedHeight() must return a positive integer.');
}
this.squarePixelWidth = data.getSquarePixelWidth();
if (!Number.isInteger(this.squarePixelWidth) || this.squarePixelWidth <= 0) {
throw new TypeError('getSquarePixelWidth() must return a positive integer.');
}
this.squarePixelHeight = data.getSquarePixelHeight();
if (!Number.isInteger(this.squarePixelHeight) || this.squarePixelHeight <= 0) {
throw new TypeError('getSquarePixelHeight() must return a positive integer.');
}
this.rotation = init.rotation ?? 0;
this.timestamp = init.timestamp;
this.duration = init.duration ?? 0;
this.colorSpace = data.getColorSpace();
}
else {
throw new TypeError('Invalid data type: Must be a BufferSource, CanvasImageSource, or VideoSampleResource.');
}
this.pixelAspectRatio = simplifyRational({
num: this.squarePixelWidth * this.codedHeight,
den: this.squarePixelHeight * this.codedWidth,
});
finalizationRegistry?.register(this, { type: 'video', data: this._data }, this);
}
/** Clones this video sample. */
clone() {
if (this._closed) {
throw new Error('VideoSample is closed.');
}
assert(this._data !== null);
if (this._data instanceof VideoSampleResource) {
return new VideoSample(this._data, {
timestamp: this.timestamp,
duration: this.duration,
rotation: this.rotation,
});
}
else if (isVideoFrame(this._data)) {
return new VideoSample(this._data.clone(), {
timestamp: this.timestamp,
duration: this.duration,
rotation: this.rotation,
});
}
else if (this._data instanceof Uint8Array) {
assert(this._layout);
return new VideoSample(this._data, {
format: this.format,
layout: this._layout,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
timestamp: this.timestamp,
duration: this.duration,
colorSpace: this.colorSpace,
rotation: this.rotation,
visibleRect: this.visibleRect,
displayWidth: this.displayWidth,
displayHeight: this.displayHeight,
});
}
else {
return new VideoSample(this._data, {
format: this.format,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
timestamp: this.timestamp,
duration: this.duration,
colorSpace: this.colorSpace,
rotation: this.rotation,
visibleRect: this.visibleRect,
displayWidth: this.displayWidth,
displayHeight: this.displayHeight,
});
}
}
/**
* Closes this video sample, releasing held resources. Video samples should be closed as soon as they are not
* needed anymore.
*/
close() {
if (this._closed) {
return;
}
finalizationRegistry?.unregister(this);
if (this._data instanceof VideoSampleResource) {
this._data._referenceCount--;
if (this._data._referenceCount === 0) {
this._data.close();
}
}
else if (isVideoFrame(this._data)) {
this._data.close();
}
else {
this._data = null; // GC that shit
}
this._closed = true;
}
/**
* Returns the number of bytes required to hold this video sample's pixel data.
*/
allocationSize(options = {}) {
validateVideoFrameCopyToOptions(options);
if (this._closed) {
throw new Error('VideoSample is closed.');
}
if ((options.format ?? this.format) == null) {
// https://github.com/Vanilagy/mediabunny/issues/267
// https://github.com/w3c/webcodecs/issues/920
throw new Error('Cannot get allocation size when format is null.');
}
if (isVideoFrame(this._data)) {
// Call the native method purely for performance
return this._data.allocationSize(options);
}
const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
return combinedLayout.allocationSize;
}
/**
* Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView.
* @returns The byte layout of the planes of the copied data.
*/
async copyTo(destination, options = {}) {
if (!isAllowSharedBufferSource(destination)) {
throw new TypeError('destination must be an ArrayBuffer or an ArrayBuffer view.');
}
validateVideoFrameCopyToOptions(options);
if (this._closed) {
throw new Error('VideoSample is closed.');
}
if ((options.format ?? this.format) == null) {
throw new Error('Cannot copy video sample data when format is null.');
}
assert(this._data !== null);
if (isVideoFrame(this._data)) {
return this._data.copyTo(destination, options);
}
// Detect non-RGB to RGB conversion
if (options.format
&& !['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(this.format)
&& ['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)) {
// RGB conversion for custom VideoSampleResource
if (this._data instanceof VideoSampleResource) {
const env_1 = { stack: [], error: void 0, hasError: false };
try {
const rgbSample = __addDisposableResource(env_1, await this._data.toRgbSample({
timestamp: this.timestamp,
duration: this.duration,
rotation: this.rotation,
}, options.colorSpace ?? 'srgb'), false);
if (!(rgbSample instanceof VideoSample)) {
throw new TypeError('toRgbSample() must return a VideoSample.');
}
if (!['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(rgbSample.format)) {
throw new Error(`Sample returned by toRgbSample was expected to have an RGB format, got`
+ ` '${rgbSample.format}' instead.`);
}
// Note that we DON'T force the RGB format to be exactly what was requested; any RGB format will do
return await rgbSample.copyTo(destination, options); // 'await' is intentional here cuz of using
}
catch (e_1) {
env_1.error = e_1;
env_1.hasError = true;
}
finally {
__disposeResources(env_1);
}
}
else {
if (typeof VideoFrame === 'undefined') {
throw new Error('For this sample, converting from a non-RGB to an RGB format requires VideoFrame to'
+ ' be defined.');
}
const tempFrame = this.toVideoFrame();
const result = await tempFrame.copyTo(destination, options);
tempFrame.close();
return result;
}
}
const combinedLayout = ParseVideoFrameCopyToOptions(this, options);
assert(this.format);
// 4. If destination.byteLength is less than combinedLayout’s allocationSize, return a promise rejected with
const destBytes = toUint8Array(destination);
if (destBytes.byteLength < combinedLayout.allocationSize) {
throw new TypeError(`Destination buffer too small. Required: ${combinedLayout.allocationSize},`
+ ` Available: ${destBytes.byteLength}`);
}
const planeConfigs = getPlaneConfigs(this.format);
let dataPlanes;
if (this._data instanceof VideoSampleResource) {
let result = this._data.getDataPlanes();
if (result instanceof Promise)
result = await result;
if (!Array.isArray(result)
|| result.some(x => !(x.data instanceof Uint8Array) || !Number.isInteger(x.stride) || x.stride < 0)) {
throw new TypeError('getDataPlanes() must return an array of objects with a Uint8Array "data" property and a'
+ ' non-negative integer "stride" property.');
}
dataPlanes = result;
}
else if (this._data instanceof Uint8Array) {
assert(this._layout);
assert(this._layout.length === planeConfigs.length);
dataPlanes = this._layout.map((planeLayout, i) => {
const height = Math.ceil(this.codedHeight / planeConfigs[i].heightDivisor);
return {
data: this._data.subarray(planeLayout.offset, planeLayout.offset + planeLayout.stride * height),
stride: planeLayout.stride,
};
});
}
else {
const canvas = this._data;
const context = canvas.getContext('2d');
assert(context);
const imageData = context.getImageData(0, 0, this.codedWidth, this.codedHeight);
dataPlanes = [{
data: toUint8Array(imageData.data),
stride: 4 * this.codedWidth,
}];
}
// Algo taken from WebCodecs spec:
// 6. Let p be a new Promise. (Implicit)
// 7. Let copyStepsQueue be the result of starting a new parallel queue. (Implicit)
// 8. Let planeLayouts be a new list.
const planeLayouts = [];
// Enqueue the following steps to copyStepsQueue: (fuck the queuing part)
// Let resource be the media resource referenced by [[resource reference]].
// (this.data)
// Let numPlanes be the number of planes as defined by [[format]].
const numPlanes = planeConfigs.length;
// Let planeIndex be 0.
// While planeIndex is less than combinedLayout’s numPlanes:
for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) {
const computedLayout = combinedLayout.computedLayouts[planeIndex];
// Let sourceStride be the stride of the plane in resource as identified by planeIndex.
const sourceStride = dataPlanes[planeIndex].stride;
const sourceData = dataPlanes[planeIndex].data;
// Let sourceOffset be the product of multiplying computedLayout’s sourceTop by sourceStride
let sourceOffset = computedLayout.sourceTop * sourceStride;
// Add computedLayout’s sourceLeftBytes to sourceOffset.
sourceOffset += computedLayout.sourceLeftBytes;
// Let destinationOffset be computedLayout’s destinationOffset.
let destinationOffset = computedLayout.destinationOffset;
// Let rowBytes be computedLayout’s sourceWidthBytes.
const rowBytes = computedLayout.sourceWidthBytes;
// Let layout be a new PlaneLayout, with offset set to destinationOffset and stride set to rowBytes.
// This is a spec error actually (https://github.com/w3c/webcodecs/issues/918)
const layout = {
offset: destinationOffset,
stride: computedLayout.destinationStride,
};
// Let row be 0.
// While row is less than computedLayout’s sourceHeight:
for (let row = 0; row < computedLayout.sourceHeight; row++) {
// Copy rowBytes bytes from resource starting at sourceOffset to destination starting
// at destinationOffset.
if (sourceOffset + rowBytes > sourceData.byteLength) {
throw new Error(`Source buffer OOB read.`);
}
if (destinationOffset + rowBytes > destBytes.byteLength) {
throw new Error(`Destination buffer OOB write.`);
}
const srcSub = sourceData.subarray(sourceOffset, sourceOffset + rowBytes);
destBytes.set(srcSub, destinationOffset);
// Increment sourceOffset by sourceStride.
sourceOffset += sourceStride;
// Increment destinationOffset by computedLayout’s destinationStride.
destinationOffset += computedLayout.destinationStride;
}
// Append layout to planeLayouts.
planeLayouts.push(layout);
}
// Now, handle converting between different RGB formats
if (options.format !== undefined) {
const needsRgbConversion = this.format.startsWith('RGB') !== options.format.startsWith('RGB');
// Going X->A requires setting the alpha to 255, going the other way doesn't since the value of X is w/e
const needsAlphaConversion = this.format.includes('X') && options.format.includes('A');
if (needsRgbConversion || needsAlphaConversion) {
// Loop over the destination bytes
for (let i = 0; i < combinedLayout.allocationSize; i += 4) {
if (needsRgbConversion) {
// Swap R with B
const r = destBytes[i];
const b = destBytes[i + 2];
destBytes[i] = b;
destBytes[i + 2] = r;
}
if (needsAlphaConversion) {
destBytes[i + 3] = 255;
}
}
}
}
// Queue a task to resolve p with planeLayouts.
return planeLayouts;
}
/**
* Converts this video sample to a VideoFrame for use with the WebCodecs API. The VideoFrame returned by this
* method *must* be closed separately from this video sample.
*/
toVideoFrame() {
if (this._closed) {
throw new Error('VideoSample is closed.');
}
assert(this._data !== null);
if (this._data instanceof VideoSampleResource) {
if (this.format === null) {
throw new Error('Cannot convert a VideoSampleResource-backed VideoSample to VideoFrame if format is null.');
}
const planes = this._data.getDataPlanes();
if (planes instanceof Promise) {
throw new Error('Cannot convert a VideoSampleResource-backed VideoSample to VideoFrame if getDataPlanes() returns'
+ ' a promise.');
}
// We can't use allocationSize since that method assumes a tight packing
const size = planes.reduce((a, b) => a + b.data.byteLength, 0);
const buffer = new Uint8Array(size);
let offset = 0;
const offsets = [];
for (const plane of planes) {
buffer.set(plane.data, offset);
offsets.push(offset);
offset += plane.data.byteLength;
}
return new VideoFrame(buffer, {
format: this.format,
layout: planes.map((x, i) => ({
offset: offsets[i],
stride: x.stride,
})),
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration,
colorSpace: this.colorSpace,
displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation
displayHeight: this.squarePixelHeight,
});
}
else if (isVideoFrame(this._data)) {
return new VideoFrame(this._data, {
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs
});
}
else if (this._data instanceof Uint8Array) {
return new VideoFrame(this._data, {
format: this.format,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration || undefined,
colorSpace: this.colorSpace,
displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation
displayHeight: this.squarePixelHeight,
});
}
else {
return new VideoFrame(this._data, {
timestamp: this.microsecondTimestamp,
duration: this.microsecondDuration || undefined,
});
}
}
draw(context, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) {
let sx = 0;
let sy = 0;
let sWidth = this.displayWidth;
let sHeight = this.displayHeight;
let dx = 0;
let dy = 0;
let dWidth = this.displayWidth;
let dHeight = this.displayHeight;
if (arg5 !== undefined) {
sx = arg1;
sy = arg2;
sWidth = arg3;
sHeight = arg4;
dx = arg5;
dy = arg6;
if (arg7 !== undefined) {
dWidth = arg7;
dHeight = arg8;
}
else {
dWidth = sWidth;
dHeight = sHeight;
}
}
else {
dx = arg1;
dy = arg2;
if (arg3 !== undefined) {
dWidth = arg3;
dHeight = arg4;
}
}
if (!((typeof CanvasRenderingContext2D !== 'undefined' && context instanceof CanvasRenderingContext2D)
|| (typeof OffscreenCanvasRenderingContext2D !== 'undefined'
&& context instanceof OffscreenCanvasRenderingContext2D))) {
throw new TypeError('context must be a CanvasRenderingContext2D or OffscreenCanvasRenderingContext2D.');
}
if (!Number.isFinite(sx)) {
throw new TypeError('sx must be a number.');
}
if (!Number.isFinite(sy)) {
throw new TypeError('sy must be a number.');
}
if (!Number.isFinite(sWidth) || sWidth < 0) {
throw new TypeError('sWidth must be a non-negative number.');
}
if (!Number.isFinite(sHeight) || sHeight < 0) {
throw new TypeError('sHeight must be a non-negative number.');
}
if (!Number.isFinite(dx)) {
throw new TypeError('dx must be a number.');
}
if (!Number.isFinite(dy)) {
throw new TypeError('dy must be a number.');
}
if (!Number.isFinite(dWidth) || dWidth < 0) {
throw new TypeError('dWidth must be a non-negative number.');
}
if (!Number.isFinite(dHeight) || dHeight < 0) {
throw new TypeError('dHeight must be a non-negative number.');
}
if (this._closed) {
throw new Error('VideoSample is closed.');
}
({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, this.rotation));
const source = this.toCanvasImageSource();
context.save();
const centerX = dx + dWidth / 2;
const centerY = dy + dHeight / 2;
context.translate(centerX, centerY);
context.rotate(this.rotation * Math.PI / 180);
const aspectRatioChange = this.rotation % 180 === 0 ? 1 : dWidth / dHeight;
// Scale to compensate for aspect ratio changes when rotated
context.scale(1 / aspectRatioChange, aspectRatioChange);
context.drawImage(source, sx, sy, sWidth, sHeight, -dWidth / 2, -dHeight / 2, dWidth, dHeight);
context.restore();
}
/**
* Draws the sample in the middle of the canvas corresponding to the context with the specified fit behavior.
*/
drawWithFit(context, options) {
if (!((typeof CanvasRenderingContext2D !== 'undefined' && context instanceof CanvasRenderingContext2D)
|| (typeof OffscreenCanvasRenderingContext2D !== 'undefined'
&& context instanceof OffscreenCanvasRenderingContext2D))) {
throw new TypeError('context must be a CanvasRenderingContext2D or OffscreenCanvasRenderingContext2D.');
}
if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.');
}
if (!['fill', 'contain', 'cover'].includes(options.fit)) {
throw new TypeError('options.fit must be \'fill\', \'contain\', or \'cover\'.');
}
if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) {
throw new TypeError('options.rotation, when provided, must be 0, 90, 180, or 270.');
}
if (options.crop !== undefined) {
validateCropRectangle(options.crop, 'options.');
}
const canvasWidth = context.canvas.width;
const canvasHeight = context.canvas.height;
const rotation = options.rotation ?? this.rotation;
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [this.squarePixelWidth, this.squarePixelHeight]
: [this.squarePixelHeight, this.squarePixelWidth];
let finalCrop = options.crop;
if (finalCrop) {
finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
}
// These variables specify where the final sample will be drawn on the canvas
let dx;
let dy;
let newWidth;
let newHeight;
const { sx, sy, sWidth, sHeight } = this._rotateSourceRegion(options.crop?.left ?? 0, options.crop?.top ?? 0, options.crop?.width ?? rotatedWidth, options.crop?.height ?? rotatedHeight, rotation);
if (options.fit === 'fill') {
dx = 0;
dy = 0;
newWidth = canvasWidth;
newHeight = canvasHeight;
}
else {
const [sampleWidth, sampleHeight] = options.crop
? [options.crop.width, options.crop.height]
: [rotatedWidth, rotatedHeight];
const scale = options.fit === 'contain'
? Math.min(canvasWidth / sampleWidth, canvasHeight / sampleHeight)
: Math.max(canvasWidth / sampleWidth, canvasHeight / sampleHeight);
newWidth = sampleWidth * scale;
newHeight = sampleHeight * scale;
dx = (canvasWidth - newWidth) / 2;
dy = (canvasHeight - newHeight) / 2;
}
context.save();
const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
context.translate(canvasWidth / 2, canvasHeight / 2);
context.rotate(rotation * Math.PI / 180);
// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
// don't need to think about how those dimensions change after the rotation
context.scale(1 / aspectRatioChange, aspectRatioChange);
context.translate(-canvasWidth / 2, -canvasHeight / 2);
// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
// ourselves here
context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight);
context.restore();
}
/** @internal */
_rotateSourceRegion(sx, sy, sWidth, sHeight, rotation) {
// The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is
// stored. Therefore, we must map these back onto the original, pre-rotation image.
if (rotation === 90) {
[sx, sy, sWidth, sHeight] = [
sy,
this.squarePixelHeight - sx - sWidth,
sHeight,
sWidth,
];
}
else if (rotation === 180) {
[sx, sy] = [
this.squarePixelWidth - sx - sWidth,
this.squarePixelHeight - sy - sHeight,
];
}
else if (rotation === 270) {
[sx, sy, sWidth, sHeight] = [
this.squarePixelWidth - sy - sHeight,
sx,
sHeight,
sWidth,
];
}
return { sx, sy, sWidth, sHeight };
}
/**
* Converts this video sample to a
* [`CanvasImageSource`](https://udn.realityripple.com/docs/Web/API/CanvasImageSource) for drawing to a canvas.
*
* You must use the value returned by this method immediately, as any VideoFrame created internally may
* automatically be closed in the next microtask.
*/
toCanvasImageSource() {
if (this._closed) {
throw new Error('VideoSample is closed.');
}
assert(this._data !== null);
if (this._data instanceof VideoSampleResource || this._data instanceof Uint8Array) {
// Requires VideoFrame to be defined
const videoFrame = this.toVideoFrame();
queueMicrotask(() => videoFrame.close()); // Let's automatically close the frame in the next microtask
return videoFrame;
}
else {
return this._data;
}
}
/**
* Transform this video sample to a new video sample given the options. Can be used to resize, rotate, and crop
* the sample.
*
* In non-browser environments, this method will not work by default. To make it work, register a custom
* transformer function via {@link registerVideoSampleTransformer}.
*/
async transform(options) {
if (!options || typeof options !== 'object') {
throw new TypeError('options must be an object.');
}
if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
throw new TypeError('options.width, when provided, must be a positive integer.');
}
if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
throw new TypeError('options.height, when provided, must be a positive integer.');
}
if (options.roundDimensionsTo !== undefined
&& (!Number.isInteger(options.roundDimensionsTo) || options.roundDimensionsTo <= 0)) {
throw new TypeError('options.roundDimensionsTo, when provided, must be a positive integer.');
}
if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
}
if (options.width !== undefined
&& options.height !== undefined
&& options.fit === undefined) {
throw new TypeError('When both options.width and options.height are provided, options.fit must also be provided.');
}
if (options.rotate !== undefined && ![0, 90, 180, 270].includes(options.rotate)) {
throw new TypeError('options.rotate, when provided, must be 0, 90, 180 or 270.');
}
if (options.crop !== undefined) {
validateCropRectangle(options.crop, 'options.');
}
if (options.alpha !== undefined && !['keep', 'discard'].includes(options.alpha)) {
throw new TypeError('options.alpha, when provided, must be \'keep\' or \'discard\'.');
}
const rotation = normalizeRotation(this.rotation + (options.rotate ?? 0));
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
? [this.squarePixelWidth, this.squarePixelHeight]
: [this.squarePixelHeight, this.squarePixelWidth];
// Clamp crop rectangle to the rotated video dimensions
let finalCrop = options.crop;
if (finalCrop) {
finalCrop = clampCropRectangle(finalCrop, rotatedWidth, rotatedHeight);
}
const cropWidth = finalCrop ? finalCrop.width : rotatedWidth;
const cropHeight = finalCrop ? finalCrop.height : rotatedHeight;
const originalAspectRatio = cropWidth / cropHeight;
let targetWidth;
let targetHeight;
if (options.width !== undefined && options.height === undefined) {
targetWidth = options.width;
targetHeight = targetWidth / originalAspectRatio;
}
else if (options.width === undefined && options.height !== undefined) {
targetHeight = options.height;
targetWidth = targetHeight * originalAspectRatio;
}
else if (options.width !== undefined && options.height !== undefined) {
targetWidth = options.width;
targetHeight = options.height;
}
else {
targetWidth = cropWidth;
targetHeight = cropHeight;
}
targetWidth = roundToMultiple(targetWidth, options.roundDimensionsTo ?? 1);
targetHeight = roundToMultiple(targetHeight, options.roundDimensionsTo ?? 1);
const description = {
width: targetWidth,
height: targetHeight,
fit: options.fit ?? 'fill',
rotation,
crop: finalCrop ?? {
left: 0,
top: 0,
width: rotatedWidth,
height: rotatedHeight,
},
alpha: options.alpha ?? 'keep',
};
// Description's finalized; let's see if a registered transformer wants to handle it
for (const transformer of registeredVideoSampleTransformers) {
let result = transformer(this, description);
if (result instanceof Promise)
result = await result;
if (result !== null) {
return result;
}
}
// We need to handle it ourselves, and we use canvases to do it
let canvas = null;
let canvasIsNew = false;
for (const entry of transformationCanvasCache)