@webav/av-cliper
Version:
WebCodecs-based, combine video, audio, images, text, with animation support 基于 WebCodecs 合成 视频、音频、图片、文字，支持动画
1,805 lines (1,800 loc) • 84.1 kB
JavaScript
import F from "@webav/mp4box.js";
import { workerTimer as ut, Log as C, autoReadStream as U, file2stream as Z, EventTool as N, recodemux as ft } from "@webav/internal-utils";
import { Log as Se } from "@webav/internal-utils";
import * as mt from "wave-resampler";
import { tmpfile as _, write as O } from "opfs-tools";
const pt = `#version 300 es
layout (location = 0) in vec4 a_position;
layout (location = 1) in vec2 a_texCoord;
out vec2 v_texCoord;
void main () {
gl_Position = a_position;
v_texCoord = a_texCoord;
}
`, wt = `#version 300 es
precision mediump float;
out vec4 FragColor;
in vec2 v_texCoord;
uniform sampler2D frameTexture;
uniform vec3 keyColor;
// 色度的相似度计算
uniform float similarity;
// 透明度的平滑度计算
uniform float smoothness;
// 降低绿幕饱和度，提高抠图准确度
uniform float spill;
vec2 RGBtoUV(vec3 rgb) {
return vec2(
rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,
rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5
);
}
void main() {
// 获取当前像素的rgba值
vec4 rgba = texture(frameTexture, v_texCoord);
// 计算当前像素与绿幕像素的色度差值
vec2 chromaVec = RGBtoUV(rgba.rgb) - RGBtoUV(keyColor);
// 计算当前像素与绿幕像素的色度距离（向量长度）, 越相像则色度距离越小
float chromaDist = sqrt(dot(chromaVec, chromaVec));
// 设置了一个相似度阈值，baseMask为负，则表明是绿幕，为正则表明不是绿幕
float baseMask = chromaDist - similarity;
// 如果baseMask为负数，fullMask等于0；baseMask为正数，越大，则透明度越低
float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5);
rgba.a = fullMask; // 设置透明度
// 如果baseMask为负数，spillVal等于0；baseMask为整数，越小，饱和度越低
float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5);
float desat = clamp(rgba.r * 0.2126 + rgba.g * 0.7152 + rgba.b * 0.0722, 0., 1.); // 计算当前像素的灰度值
rgba.rgb = mix(vec3(desat, desat, desat), rgba.rgb, spillVal);
FragColor = rgba;
}
`, yt = [-1, 1, -1, -1, 1, -1, 1, -1, 1, 1, -1, 1], gt = [0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1];
function bt(s, t, e) {
const i = G(s, s.VERTEX_SHADER, t), n = G(s, s.FRAGMENT_SHADER, e), a = s.createProgram();
if (s.attachShader(a, i), s.attachShader(a, n), s.linkProgram(a), !s.getProgramParameter(a, s.LINK_STATUS))
throw Error(
s.getProgramInfoLog(a) ?? "Unable to initialize the shader program"
);
return a;
}
function G(s, t, e) {
const i = s.createShader(t);
if (s.shaderSource(i, e), s.compileShader(i), !s.getShaderParameter(i, s.COMPILE_STATUS)) {
const n = s.getShaderInfoLog(i);
throw s.deleteShader(i), Error(n ?? "An error occurred compiling the shaders");
}
return i;
}
function xt(s, t, e) {
s.bindTexture(s.TEXTURE_2D, e), s.texImage2D(s.TEXTURE_2D, 0, s.RGBA, s.RGBA, s.UNSIGNED_BYTE, t), s.drawArrays(s.TRIANGLES, 0, 6);
}
function Ct(s) {
const t = s.createTexture();
if (t == null) throw Error("Create WebGL texture error");
s.bindTexture(s.TEXTURE_2D, t);
const e = 0, i = s.RGBA, n = 1, a = 1, r = 0, o = s.RGBA, c = s.UNSIGNED_BYTE, l = new Uint8Array([0, 0, 255, 255]);
return s.texImage2D(
s.TEXTURE_2D,
e,
i,
n,
a,
r,
o,
c,
l
), s.texParameteri(s.TEXTURE_2D, s.TEXTURE_MAG_FILTER, s.LINEAR), s.texParameteri(s.TEXTURE_2D, s.TEXTURE_MIN_FILTER, s.LINEAR), s.texParameteri(s.TEXTURE_2D, s.TEXTURE_WRAP_S, s.CLAMP_TO_EDGE), s.texParameteri(s.TEXTURE_2D, s.TEXTURE_WRAP_T, s.CLAMP_TO_EDGE), t;
}
function vt(s) {
const t = "document" in globalThis ? globalThis.document.createElement("canvas") : new OffscreenCanvas(s.width, s.height);
t.width = s.width, t.height = s.height;
const e = t.getContext("webgl2", {
premultipliedAlpha: !1,
alpha: !0
});
if (e == null) throw Error("Cant create gl context");
const i = bt(e, pt, wt);
e.useProgram(i), e.uniform3fv(
e.getUniformLocation(i, "keyColor"),
s.keyColor.map((c) => c / 255)
), e.uniform1f(
e.getUniformLocation(i, "similarity"),
s.similarity
), e.uniform1f(
e.getUniformLocation(i, "smoothness"),
s.smoothness
), e.uniform1f(e.getUniformLocation(i, "spill"), s.spill);
const n = e.createBuffer();
e.bindBuffer(e.ARRAY_BUFFER, n), e.bufferData(e.ARRAY_BUFFER, new Float32Array(yt), e.STATIC_DRAW);
const a = e.getAttribLocation(i, "a_position");
e.vertexAttribPointer(
a,
2,
e.FLOAT,
!1,
Float32Array.BYTES_PER_ELEMENT * 2,
0
), e.enableVertexAttribArray(a);
const r = e.createBuffer();
e.bindBuffer(e.ARRAY_BUFFER, r), e.bufferData(
e.ARRAY_BUFFER,
new Float32Array(gt),
e.STATIC_DRAW
);
const o = e.getAttribLocation(i, "a_texCoord");
return e.vertexAttribPointer(
o,
2,
e.FLOAT,
!1,
Float32Array.BYTES_PER_ELEMENT * 2,
0
), e.enableVertexAttribArray(o), e.pixelStorei(e.UNPACK_FLIP_Y_WEBGL, 1), { cvs: t, gl: e };
}
function St(s) {
return s instanceof VideoFrame ? { width: s.codedWidth, height: s.codedHeight } : { width: s.width, height: s.height };
}
function Tt(s) {
const e = new OffscreenCanvas(1, 1).getContext("2d");
e.drawImage(s, 0, 0);
const {
data: [i, n, a]
} = e.getImageData(0, 0, 1, 1);
return [i, n, a];
}
const we = (s) => {
let t = null, e = null, i = s.keyColor, n = null;
return async (a) => {
if ((t == null || e == null || n == null) && (i == null && (i = Tt(a)), { cvs: t, gl: e } = vt({
...St(a),
keyColor: i,
...s
}), n = Ct(e)), xt(e, a, n), globalThis.VideoFrame != null && a instanceof globalThis.VideoFrame) {
const r = new VideoFrame(t, {
alpha: "keep",
timestamp: a.timestamp,
duration: a.duration ?? void 0
});
return a.close(), r;
}
return createImageBitmap(t, {
imageOrientation: a instanceof ImageBitmap ? "flipY" : "none"
});
};
};
function At(s) {
return document.createElement(s);
}
function Ft(s) {
var t = "", e = new Uint8Array(s), i = e.byteLength;
for (let n = 0; n < i; n++)
t += String.fromCharCode(e[n]);
return window.btoa(t);
}
async function kt(s, t, e = {}) {
const i = At("pre");
i.style.cssText = `margin: 0; ${t}; position: fixed;`, i.textContent = s, document.body.appendChild(i), e.onCreated?.(i);
const n = "TMP_FONT_NAME_" + crypto.randomUUID();
let a = null;
e.font != null && (i.style.fontFamily = n, a = new FontFace(n, `url(${e.font.url})`), await a.load(), document.fonts.add(a), await document.fonts.ready);
const { width: r, height: o } = i.getBoundingClientRect();
i.remove(), a != null && document.fonts.delete(a);
const c = new Image();
c.width = r, c.height = o;
const l = e.font == null ? "" : `
@font-face {
font-family: '${n}';
src: url('data:font/woff2;base64,${Ft(await (await fetch(e.font.url)).arrayBuffer())}') format('woff2');
}
`, h = `
<svg xmlns="http://www.w3.org/2000/svg" width="${r}" height="${o}">
<style>
${l}
</style>
<foreignObject width="100%" height="100%">
<div xmlns="http://www.w3.org/1999/xhtml">${i.outerHTML}</div>
</foreignObject>
</svg>
`.replace(/\t/g, "").replace(/#/g, "%23");
return c.src = `data:image/svg+xml;charset=utf-8,${h}`, await new Promise((u) => {
c.onload = u;
}), c;
}
async function ye(s, t, e = {}) {
const i = await kt(s, t, e), n = new OffscreenCanvas(i.width, i.height);
return n.getContext("2d")?.drawImage(i, 0, 0, i.width, i.height), await createImageBitmap(n);
}
function It(s) {
const t = new Float32Array(
s.map((i) => i.length).reduce((i, n) => i + n)
);
let e = 0;
for (const i of s)
t.set(i, e), e += i.length;
return t;
}
function tt(s) {
const t = [];
for (let e = 0; e < s.length; e += 1)
for (let i = 0; i < s[e].length; i += 1)
t[i] == null && (t[i] = []), t[i].push(s[e][i]);
return t.map(It);
}
function et(s) {
if (s.format === "f32-planar") {
const t = [];
for (let e = 0; e < s.numberOfChannels; e += 1) {
const i = s.allocationSize({ planeIndex: e }), n = new ArrayBuffer(i);
s.copyTo(n, { planeIndex: e }), t.push(new Float32Array(n));
}
return t;
} else if (s.format === "f32") {
const t = new ArrayBuffer(s.allocationSize({ planeIndex: 0 }));
return s.copyTo(t, { planeIndex: 0 }), Et(new Float32Array(t), s.numberOfChannels);
} else if (s.format === "s16") {
const t = new ArrayBuffer(s.allocationSize({ planeIndex: 0 }));
return s.copyTo(t, { planeIndex: 0 }), Rt(new Int16Array(t), s.numberOfChannels);
}
throw Error("Unsupported audio data format");
}
function Rt(s, t) {
const e = s.length / t, i = Array.from(
{ length: t },
() => new Float32Array(e)
);
for (let n = 0; n < e; n++)
for (let a = 0; a < t; a++) {
const r = s[n * t + a];
i[a][n] = r / 32768;
}
return i;
}
function Et(s, t) {
const e = s.length / t, i = Array.from(
{ length: t },
() => new Float32Array(e)
);
for (let n = 0; n < e; n++)
for (let a = 0; a < t; a++)
i[a][n] = s[n * t + a];
return i;
}
function $(s) {
return Array(s.numberOfChannels).fill(0).map((t, e) => s.getChannelData(e));
}
async function Dt(s, t) {
const e = {
type: t,
data: s
}, i = new ImageDecoder(e);
await Promise.all([i.completed, i.tracks.ready]);
let n = i.tracks.selectedTrack?.frameCount ?? 1;
const a = [];
for (let r = 0; r < n; r += 1)
a.push((await i.decode({ frameIndex: r })).image);
return a;
}
function j(s) {
const t = Math.max(...s.map((i) => i[0]?.length ?? 0)), e = new Float32Array(t * 2);
for (let i = 0; i < t; i++) {
let n = 0, a = 0;
for (let r = 0; r < s.length; r++) {
const o = s[r][0]?.[i] ?? 0, c = s[r][1]?.[i] ?? o;
n += o, a += c;
}
e[i] = n, e[i + t] = a;
}
return e;
}
async function Pt(s, t, e) {
const i = s.length, n = Array(e.chanCount).fill(0).map(() => new Float32Array(0));
if (i === 0) return n;
const a = Math.max(...s.map((l) => l.length));
if (a === 0) return n;
if (globalThis.OfflineAudioContext == null)
return s.map(
(l) => new Float32Array(
mt.resample(l, t, e.rate, {
method: "sinc",
LPF: !1
})
)
);
const r = new globalThis.OfflineAudioContext(
e.chanCount,
a * e.rate / t,
e.rate
), o = r.createBufferSource(), c = r.createBuffer(i, a, t);
return s.forEach((l, h) => c.copyToChannel(l, h)), o.buffer = c, o.connect(r.destination), o.start(), $(await r.startRendering());
}
function H(s) {
return new Promise((t) => {
const e = ut(() => {
e(), t();
}, s);
});
}
function z(s, t, e) {
const i = e - t, n = new Float32Array(i);
let a = 0;
for (; a < i; )
n[a] = s[(t + a) % s.length], a += 1;
return n;
}
function it(s, t) {
const e = Math.floor(s.length / t), i = new Float32Array(e);
for (let n = 0; n < e; n++) {
const a = n * t, r = Math.floor(a), o = a - r;
r + 1 < s.length ? i[n] = s[r] * (1 - o) + s[r + 1] * o : i[n] = s[r];
}
return i;
}
const x = {
sampleRate: 48e3,
channelCount: 2,
codec: "mp4a.40.2"
};
function W(s, t) {
const e = t.videoTracks[0], i = {};
if (e != null) {
const a = Mt(s.getTrackById(e.id))?.buffer, { descKey: r, type: o } = e.codec.startsWith("avc1") ? { descKey: "avcDecoderConfigRecord", type: "avc1" } : e.codec.startsWith("hvc1") ? { descKey: "hevcDecoderConfigRecord", type: "hvc1" } : { descKey: "", type: "" };
r !== "" && (i.videoTrackConf = {
timescale: e.timescale,
duration: e.duration,
width: e.video.width,
height: e.video.height,
brands: t.brands,
type: o,
[r]: a
}), i.videoDecoderConf = {
codec: e.codec,
codedHeight: e.video.height,
codedWidth: e.video.width,
description: a
};
}
const n = t.audioTracks[0];
if (n != null) {
const a = Bt(s), r = a == null ? {} : _t(a);
i.audioTrackConf = {
timescale: n.timescale,
samplerate: r.sampleRate ?? n.audio.sample_rate,
channel_count: r.numberOfChannels ?? n.audio.channel_count,
hdlr: "soun",
type: n.codec.startsWith("mp4a") ? "mp4a" : n.codec,
description: a
}, i.audioDecoderConf = {
codec: r.codec ?? x.codec,
numberOfChannels: r.numberOfChannels ?? n.audio.channel_count,
sampleRate: r.sampleRate ?? n.audio.sample_rate
};
}
return i;
}
function Mt(s) {
for (const t of s.mdia.minf.stbl.stsd.entries) {
const e = t.avcC ?? t.hvcC ?? t.av1C ?? t.vpcC;
if (e != null) {
const i = new F.DataStream(
void 0,
0,
F.DataStream.BIG_ENDIAN
);
return e.write(i), new Uint8Array(i.buffer.slice(8));
}
}
}
function Bt(s, t = "mp4a") {
return s.moov?.traks.map((i) => i.mdia.minf.stbl.stsd.entries).flat().find(({ type: i }) => i === t)?.esds;
}
function _t(s) {
let t = "mp4a";
const e = s.esd.descs[0];
if (e == null) return {};
t += "." + e.oti.toString(16);
const i = e.descs[0];
if (i == null)
return t.endsWith("40") && (t += ".2"), { codec: t };
const n = (i.data[0] & 248) >> 3;
t += "." + n;
const [a, r] = i.data, o = ((a & 7) << 1) + (r >> 7), c = (r & 127) >> 3;
return {
codec: t,
sampleRate: [
96e3,
88200,
64e3,
48e3,
44100,
32e3,
24e3,
22050,
16e3,
12e3,
11025,
8e3,
7350
][o],
numberOfChannels: c
};
}
async function Ot(s, t, e) {
const i = F.createFile(!1);
i.onReady = (a) => {
t({ mp4boxFile: i, info: a });
const r = a.videoTracks[0]?.id;
r != null && i.setExtractionOptions(r, "video", { nbSamples: 100 });
const o = a.audioTracks[0]?.id;
o != null && i.setExtractionOptions(o, "audio", { nbSamples: 100 }), i.start();
}, i.onSamples = e, await n();
async function n() {
let a = 0;
const r = 30 * 1024 * 1024;
for (; ; ) {
const o = await s.read(r, {
at: a
});
if (o.byteLength === 0) break;
o.fileStart = a;
const c = i.appendBuffer(o);
if (c == null) break;
a = c;
}
i.stop();
}
}
function zt(s) {
if (s?.length !== 9) return {};
const t = new Int32Array(s.buffer), e = t[0] / 65536, i = t[1] / 65536, n = t[3] / 65536, a = t[4] / 65536, r = t[6] / 65536, o = t[7] / 65536, c = t[8] / (1 << 30), l = Math.sqrt(e * e + n * n), h = Math.sqrt(i * i + a * a), u = Math.atan2(n, e), d = u * 180 / Math.PI;
return {
scaleX: l,
scaleY: h,
rotationRad: u,
rotationDeg: d,
translateX: r,
translateY: o,
perspective: c
};
}
function Vt(s, t, e) {
const i = (Math.round(e / 90) * 90 + 360) % 360;
if (i === 0) return (c) => c;
const n = i === 90 || i === 270 ? t : s, a = i === 90 || i === 270 ? s : t, r = new OffscreenCanvas(n, a), o = r.getContext("2d");
return o.translate(n / 2, a / 2), o.rotate(-i * Math.PI / 180), o.translate(-s / 2, -t / 2), (c) => {
if (c == null) return null;
o.drawImage(c, 0, 0);
const l = new VideoFrame(r, {
timestamp: c.timestamp,
duration: c.duration ?? void 0
});
return c.close(), l;
};
}
let X = 0;
function B(s) {
return s.kind === "file" && s.createReader instanceof Function;
}
class I {
#t = X++;
#n = C.create(`MP4Clip id:${this.#t},`);
ready;
#e = !1;
#s = {
// 微秒
duration: 0,
width: 0,
height: 0,
audioSampleRate: 0,
audioChanCount: 0
};
get meta() {
return { ...this.#s };
}
#a;
/** 存储视频头（box: ftyp, moov）的二进制数据 */
#r = [];
/**
* 提供视频头（box: ftyp, moov）的二进制数据
* 使用任意 mp4 demxer 解析即可获得详细的视频信息
* 单元测试包含使用 mp4box.js 解析示例代码
*/
async getFileHeaderBinData() {
await this.ready;
const t = await this.#a.getOriginFile();
if (t == null) throw Error("MP4Clip localFile is not origin file");
return await new Blob(
this.#r.map(
({ start: e, size: i }) => t.slice(e, e + i)
)
).arrayBuffer();
}
/**存储视频平移旋转信息，目前只还原旋转 */
#i = {
perspective: 1,
rotationRad: 0,
rotationDeg: 0,
scaleX: 1,
scaleY: 1,
translateX: 0,
translateY: 0
};
#o = (t) => t;
#l = 1;
#c = [];
#d = [];
#m = null;
#u = null;
#f = {
video: null,
audio: null
};
#h = { audio: !0 };
constructor(t, e = {}) {
if (!(t instanceof ReadableStream) && !B(t) && !Array.isArray(t.videoSamples))
throw Error("Illegal argument");
this.#h = { audio: !0, ...e }, this.#l = typeof e.audio == "object" && "volume" in e.audio ? e.audio.volume : 1;
const i = async (n) => (await O(this.#a, n), this.#a);
this.#a = B(t) ? t : "localFile" in t ? t.localFile : _(), this.ready = (t instanceof ReadableStream ? i(t).then(
(n) => J(n, this.#h)
) : B(t) ? J(t, this.#h) : Promise.resolve(t)).then(
async ({
videoSamples: n,
audioSamples: a,
decoderConf: r,
headerBoxPos: o,
parsedMatrix: c
}) => {
this.#c = n, this.#d = a, this.#f = r, this.#r = o, this.#i = c;
const { videoFrameFinder: l, audioFrameFinder: h } = Ut(
{
video: r.video == null ? null : {
...r.video,
hardwareAcceleration: this.#h.__unsafe_hardwareAcceleration__
},
audio: r.audio
},
await this.#a.createReader(),
n,
a,
this.#h.audio !== !1 ? this.#l : 0
);
this.#m = l, this.#u = h;
const { codedWidth: u, codedHeight: d } = r.video ?? {};
return u && d && (this.#o = Vt(
u,
d,
c.rotationDeg
)), this.#s = Lt(
r,
n,
a,
c.rotationDeg
), this.#n.info("MP4Clip meta:", this.#s), { ...this.#s };
}
);
}
/**
* 拦截 {@link MP4Clip.tick} 方法返回的数据，用于对图像、音频数据二次处理
* @param time 调用 tick 的时间
* @param tickRet tick 返回的数据
*
* @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)
*/
tickInterceptor = async (t, e) => e;
/**
* 获取素材指定时刻的图像帧、音频数据
* @param time 微秒
*/
async tick(t) {
if (t >= this.#s.duration)
return await this.tickInterceptor(t, {
audio: await this.#u?.find(t) ?? [],
state: "done"
});
const [e, i] = await Promise.all([
this.#u?.find(t) ?? [],
this.#m?.find(t).then(this.#o)
]);
return i == null ? await this.tickInterceptor(t, {
audio: e,
state: "success"
}) : await this.tickInterceptor(t, {
video: i,
audio: e,
state: "success"
});
}
#p = new AbortController();
/**
* 生成缩略图，默认每个关键帧生成一个 100px 宽度的缩略图。
*
* @param imgWidth 缩略图宽度，默认 100
* @param opts Partial<ThumbnailOpts>
* @returns Promise<Array<{ ts: number; img: Blob }>>
*/
async thumbnails(t = 100, e) {
this.#p.abort(), this.#p = new AbortController();
const i = this.#p.signal;
await this.ready;
const n = "generate thumbnails aborted";
if (i.aborted) throw Error(n);
const { width: a, height: r } = this.#s, o = Xt(
t,
Math.round(r * (t / a)),
{ quality: 0.1, type: "image/png" }
);
return new Promise(
async (c, l) => {
let h = [];
const u = this.#f.video;
if (u == null || this.#c.length === 0) {
d();
return;
}
i.addEventListener("abort", () => {
l(Error(n));
});
async function d() {
i.aborted || c(
await Promise.all(
h.map(async (p) => ({
ts: p.ts,
img: await p.img
}))
)
);
}
function y(p) {
h.push({
ts: p.timestamp,
img: o(p)
});
}
const { start: m = 0, end: f = this.#s.duration, step: w } = e ?? {};
if (w) {
let p = m;
const g = new nt(
await this.#a.createReader(),
this.#c,
{
...u,
hardwareAcceleration: this.#h.__unsafe_hardwareAcceleration__
}
);
for (; p <= f && !i.aborted; ) {
const b = await g.find(p);
b && y(b), p += w;
}
g.destroy(), d();
} else
await Jt(
this.#c,
this.#a,
u,
i,
{ start: m, end: f },
(p, g) => {
p != null && y(p), g && d();
}
);
}
);
}
async split(t) {
if (await this.ready, t <= 0 || t >= this.#s.duration)
throw Error('"time" out of bounds');
const [e, i] = Yt(
this.#c,
t
), [n, a] = Gt(
this.#d,
t
), r = new I(
{
localFile: this.#a,
videoSamples: e ?? [],
audioSamples: n ?? [],
decoderConf: this.#f,
headerBoxPos: this.#r,
parsedMatrix: this.#i
},
this.#h
), o = new I(
{
localFile: this.#a,
videoSamples: i ?? [],
audioSamples: a ?? [],
decoderConf: this.#f,
headerBoxPos: this.#r,
parsedMatrix: this.#i
},
this.#h
);
return await Promise.all([r.ready, o.ready]), [r, o];
}
async clone() {
await this.ready;
const t = new I(
{
localFile: this.#a,
videoSamples: [...this.#c],
audioSamples: [...this.#d],
decoderConf: this.#f,
headerBoxPos: this.#r,
parsedMatrix: this.#i
},
this.#h
);
return await t.ready, t.tickInterceptor = this.tickInterceptor, t;
}
/**
* 拆分 MP4Clip 为仅包含视频轨道和音频轨道的 MP4Clip
* @returns Mp4CLip[]
*/
async splitTrack() {
await this.ready;
const t = [];
if (this.#c.length > 0) {
const e = new I(
{
localFile: this.#a,
videoSamples: [...this.#c],
audioSamples: [],
decoderConf: {
video: this.#f.video,
audio: null
},
headerBoxPos: this.#r,
parsedMatrix: this.#i
},
this.#h
);
await e.ready, e.tickInterceptor = this.tickInterceptor, t.push(e);
}
if (this.#d.length > 0) {
const e = new I(
{
localFile: this.#a,
videoSamples: [],
audioSamples: [...this.#d],
decoderConf: {
audio: this.#f.audio,
video: null
},
headerBoxPos: this.#r,
parsedMatrix: this.#i
},
this.#h
);
await e.ready, e.tickInterceptor = this.tickInterceptor, t.push(e);
}
return t;
}
destroy() {
this.#e || (this.#n.info("MP4Clip destroy"), this.#e = !0, this.#m?.destroy(), this.#u?.destroy());
}
}
function Lt(s, t, e, i) {
const n = {
duration: 0,
width: 0,
height: 0,
audioSampleRate: 0,
audioChanCount: 0
};
if (s.video != null && t.length > 0) {
n.width = s.video.codedWidth ?? 0, n.height = s.video.codedHeight ?? 0;
const o = (Math.round(i / 90) * 90 + 360) % 360;
(o === 90 || o === 270) && ([n.width, n.height] = [n.height, n.width]);
}
s.audio != null && e.length > 0 && (n.audioSampleRate = x.sampleRate, n.audioChanCount = x.channelCount);
let a = 0, r = 0;
if (t.length > 0)
for (let o = t.length - 1; o >= 0; o--) {
const c = t[o];
if (!c.deleted) {
a = c.cts + c.duration;
break;
}
}
if (e.length > 0) {
const o = e.at(-1);
r = o.cts + o.duration;
}
return n.duration = Math.max(a, r), n;
}
function Ut(s, t, e, i, n) {
return {
audioFrameFinder: n === 0 || s.audio == null || i.length === 0 ? null : new $t(
t,
i,
s.audio,
{
volume: n,
targetSampleRate: x.sampleRate
}
),
videoFrameFinder: s.video == null || e.length === 0 ? null : new nt(
t,
e,
s.video
)
};
}
async function J(s, t = {}) {
let e = null;
const i = { video: null, audio: null };
let n = [], a = [], r = [];
const o = {
perspective: 1,
rotationRad: 0,
rotationDeg: 0,
scaleX: 1,
scaleY: 1,
translateX: 0,
translateY: 0
};
let c = -1, l = -1;
const h = await s.createReader();
await Ot(
h,
async (d) => {
e = d.info;
const y = d.mp4boxFile.ftyp;
r.push({ start: y.start, size: y.size });
const m = d.mp4boxFile.moov;
r.push({ start: m.start, size: m.size }), Object.assign(o, zt(e.videoTracks[0]?.matrix));
let { videoDecoderConf: f, audioDecoderConf: w } = W(
d.mp4boxFile,
d.info
);
if (i.video = f ?? null, i.audio = w ?? null, f == null && w == null && C.error("MP4Clip no video and audio track"), w != null) {
const { supported: p } = await AudioDecoder.isConfigSupported(w);
p || C.error(`MP4Clip audio codec is not supported: ${w.codec}`);
}
if (f != null) {
const { supported: p } = await VideoDecoder.isConfigSupported(f);
p || C.error(`MP4Clip video codec is not supported: ${f.codec}`);
}
C.info(
"mp4BoxFile moov ready",
{
...d.info,
tracks: null,
videoTracks: null,
audioTracks: null
},
i
);
},
(d, y, m) => {
if (y === "video") {
c === -1 && (c = m[0].dts);
for (const f of m)
n.push(Q(f, c, "video"));
} else if (y === "audio" && t.audio) {
l === -1 && (l = m[0].dts);
for (const f of m)
a.push(Q(f, l, "audio"));
}
}
), await h.close();
const u = n.at(-1) ?? a.at(-1);
if (e == null)
throw Error("MP4Clip stream is done, but not emit ready");
if (u == null)
throw Error("MP4Clip stream not contain any sample");
return L(n), C.info("mp4 stream parsed"), {
videoSamples: n,
audioSamples: a,
decoderConf: i,
headerBoxPos: r,
parsedMatrix: o
};
}
function Q(s, t = 0, e) {
let i = s.offset;
const n = e === "video" && s.is_sync ? jt(s.data, s.description.type) : -1;
let a = s.size;
return n > 0 && (i += n, a -= n), {
...s,
is_idr: n >= 0,
offset: i,
size: a,
cts: (s.cts - t) / s.timescale * 1e6,
dts: (s.dts - t) / s.timescale * 1e6,
duration: s.duration / s.timescale * 1e6,
timescale: 1e6,
// 音频数据量可控，直接保存在内存中
data: e === "video" ? null : s.data
};
}
class nt {
constructor(t, e, i) {
this.localFileReader = t, this.samples = e, this.conf = i;
}
#t = null;
#n = 0;
#e = { abort: !1, st: performance.now() };
find = async (t) => {
(this.#t == null || this.#t.state === "closed" || t <= this.#n || t - this.#n > 3e6) && this.#h(t), this.#e.abort = !0, this.#n = t, this.#e = { abort: !1, st: performance.now() };
const e = await this.#m(t, this.#t, this.#e);
return this.#c = 0, e;
};
// fix VideoFrame duration is null
#s = 0;
#a = !1;
#r = 0;
#i = [];
#o = 0;
#l = 0;
#c = 0;
#d = !1;
#m = async (t, e, i) => {
if (e == null || e.state === "closed" || i.abort) return null;
if (this.#i.length > 0) {
const n = this.#i[0];
return t < n.timestamp ? null : (this.#i.shift(), t > n.timestamp + (n.duration ?? 0) ? (n.close(), await this.#m(t, e, i)) : (!this.#d && this.#i.length < 10 && this.#f(e).catch((a) => {
throw this.#d = !0, this.#h(t), a;
}), n));
}
if (this.#u || this.#o < this.#l && e.decodeQueueSize > 0) {
if (performance.now() - i.st > 6e3)
throw Error(
`MP4Clip.tick video timeout, ${JSON.stringify(this.#p())}`
);
this.#c += 1, await H(15);
} else {
if (this.#r >= this.samples.length)
return null;
try {
await this.#f(e);
} catch (n) {
throw this.#h(t), n;
}
}
return await this.#m(t, e, i);
};
#u = !1;
#f = async (t) => {
if (this.#u || t.decodeQueueSize > 600) return;
let e = this.#r + 1;
if (e > this.samples.length) return;
this.#u = !0;
let i = !1;
for (; e < this.samples.length; e++) {
const n = this.samples[e];
if (!i && !n.deleted && (i = !0), n.is_idr) break;
}
if (i) {
const n = this.samples.slice(this.#r, e);
if (n[0]?.is_idr !== !0)
C.warn("First sample not idr frame");
else {
const a = performance.now(), r = await st(n, this.localFileReader), o = performance.now() - a;
if (o > 1e3) {
const c = n[0], l = n.at(-1), h = l.offset + l.size - c.offset;
C.warn(
`Read video samples time cost: ${Math.round(o)}ms, file chunk size: ${h}`
);
}
if (t.state === "closed") return;
this.#s = r[0]?.duration ?? 0, V(t, r, {
onDecodingError: (c) => {
if (this.#a)
throw c;
this.#o === 0 && (this.#a = !0, C.warn("Downgrade to software decode"), this.#h());
}
}), this.#l += r.length;
}
}
this.#r = e, this.#u = !1;
};
#h = (t) => {
if (this.#u = !1, this.#i.forEach((i) => i.close()), this.#i = [], t == null || t === 0)
this.#r = 0;
else {
let i = 0;
for (let n = 0; n < this.samples.length; n++) {
const a = this.samples[n];
if (a.is_idr && (i = n), !(a.cts < t)) {
this.#r = i;
break;
}
}
}
this.#l = 0, this.#o = 0, this.#t?.state !== "closed" && this.#t?.close();
const e = {
...this.conf,
...this.#a ? { hardwareAcceleration: "prefer-software" } : {}
};
this.#t = new VideoDecoder({
output: (i) => {
if (this.#o += 1, i.timestamp === -1) {
i.close();
return;
}
let n = i;
i.duration == null && (n = new VideoFrame(i, {
duration: this.#s
}), i.close()), this.#i.push(n);
},
error: (i) => {
if (i.message.includes("Codec reclaimed due to inactivity")) {
this.#t = null, C.warn(i.message);
return;
}
const n = `VideoFinder VideoDecoder err: ${i.message}, config: ${JSON.stringify(e)}, state: ${JSON.stringify(this.#p())}`;
throw C.error(n), Error(n);
}
}), this.#t.configure(e);
};
#p = () => ({
time: this.#n,
decState: this.#t?.state,
decQSize: this.#t?.decodeQueueSize,
decCusorIdx: this.#r,
sampleLen: this.samples.length,
inputCnt: this.#l,
outputCnt: this.#o,
cacheFrameLen: this.#i.length,
softDeocde: this.#a,
clipIdCnt: X,
sleepCnt: this.#c,
memInfo: at()
});
destroy = () => {
this.#t?.state !== "closed" && this.#t?.close(), this.#t = null, this.#e.abort = !0, this.#i.forEach((t) => t.close()), this.#i = [], this.localFileReader.close();
};
}
function Nt(s, t) {
for (let e = 0; e < t.length; e++) {
const i = t[e];
if (s >= i.cts && s < i.cts + i.duration)
return e;
if (i.cts > s) break;
}
return 0;
}
class $t {
constructor(t, e, i, n) {
this.localFileReader = t, this.samples = e, this.conf = i, this.#t = n.volume, this.#n = n.targetSampleRate;
}
#t = 1;
#n;
#e = null;
#s = { abort: !1, st: performance.now() };
find = async (t) => {
const e = t <= this.#a || t - this.#a > 1e5;
(this.#e == null || this.#e.state === "closed" || e) && this.#d(), e && (this.#a = t, this.#r = Nt(t, this.samples)), this.#s.abort = !0;
const i = t - this.#a;
this.#a = t, this.#s = { abort: !1, st: performance.now() };
const n = await this.#l(
Math.ceil(i * (this.#n / 1e6)),
this.#e,
this.#s
);
return this.#o = 0, n;
};
#a = 0;
#r = 0;
#i = {
frameCnt: 0,
data: []
};
#o = 0;
#l = async (t, e = null, i) => {
if (e == null || i.abort || e.state === "closed" || t === 0)
return [];
const n = this.#i.frameCnt - t;
if (n > 0)
return n < x.sampleRate / 10 && this.#c(e), K(this.#i, t);
if (e.decoding) {
if (performance.now() - i.st > 3e3)
throw i.abort = !0, Error(
`MP4Clip.tick audio timeout, ${JSON.stringify(this.#m())}`
);
this.#o += 1, await H(15);
} else {
if (this.#r >= this.samples.length - 1)
return K(this.#i, this.#i.frameCnt);
this.#c(e);
}
return this.#l(t, e, i);
};
#c = (t) => {
if (t.decodeQueueSize > 10) return;
const i = [];
let n = this.#r;
for (; n < this.samples.length; ) {
const a = this.samples[n];
if (n += 1, !a.deleted && (i.push(a), i.length >= 10))
break;
}
this.#r = n, t.decode(
i.map(
(a) => new EncodedAudioChunk({
type: "key",
timestamp: a.cts,
duration: a.duration,
data: a.data
})
)
);
};
#d = () => {
this.#a = 0, this.#r = 0, this.#i = {
frameCnt: 0,
data: []
}, this.#e?.close(), this.#e = Ht(
this.conf,
{
resampleRate: x.sampleRate,
volume: this.#t
},
(t) => {
this.#i.data.push(t), this.#i.frameCnt += t[0].length;
}
);
};
#m = () => ({
time: this.#a,
decState: this.#e?.state,
decQSize: this.#e?.decodeQueueSize,
decCusorIdx: this.#r,
sampleLen: this.samples.length,
pcmLen: this.#i.frameCnt,
clipIdCnt: X,
sleepCnt: this.#o,
memInfo: at()
});
destroy = () => {
this.#e = null, this.#s.abort = !0, this.#i = {
frameCnt: 0,
data: []
}, this.localFileReader.close();
};
}
function Ht(s, t, e) {
let i = 0, n = 0;
const a = (h) => {
if (n += 1, h.length !== 0) {
if (t.volume !== 1)
for (const u of h)
for (let d = 0; d < u.length; d++) u[d] *= t.volume;
h.length === 1 && (h = [h[0], h[0]]), e(h);
}
}, r = Wt(a), o = t.resampleRate !== s.sampleRate;
let c = new AudioDecoder({
output: (h) => {
const u = et(h);
o ? r(
() => Pt(u, h.sampleRate, {
rate: t.resampleRate,
chanCount: h.numberOfChannels
})
) : a(u), h.close();
},
error: (h) => {
h.message.includes("Codec reclaimed due to inactivity") || l("MP4Clip AudioDecoder err", h);
}
});
c.configure(s);
function l(h, u) {
const d = `${h}: ${u.message}, state: ${JSON.stringify(
{
qSize: c.decodeQueueSize,
state: c.state,
inputCnt: i,
outputCnt: n
}
)}`;
throw C.error(d), Error(d);
}
return {
decode(h) {
i += h.length;
try {
for (const u of h) c.decode(u);
} catch (u) {
l("decode audio chunk error", u);
}
},
close() {
c.state !== "closed" && c.close();
},
get decoding() {
return i > n && c.decodeQueueSize > 0;
},
get state() {
return c.state;
},
get decodeQueueSize() {
return c.decodeQueueSize;
}
};
}
function Wt(s) {
const t = [];
let e = 0;
function i(r, o) {
t[o] = r, n();
}
function n() {
const r = t[e];
r != null && (s(r), e += 1, n());
}
let a = 0;
return (r) => {
const o = a;
a += 1, r().then((c) => i(c, o)).catch((c) => i(c, o));
};
}
function K(s, t) {
const e = [new Float32Array(t), new Float32Array(t)];
let i = 0, n = 0;
for (; n < s.data.length; ) {
const [a, r] = s.data[n];
if (i + a.length > t) {
const o = t - i;
e[0].set(a.subarray(0, o), i), e[1].set(r.subarray(0, o), i), s.data[n][0] = a.subarray(o, a.length), s.data[n][1] = r.subarray(o, r.length);
break;
} else
e[0].set(a, i), e[1].set(r, i), i += a.length, n++;
}
return s.data = s.data.slice(n), s.frameCnt -= t, e;
}
async function st(s, t) {
const e = s[0], i = s.at(-1);
if (i == null) return [];
const n = i.offset + i.size - e.offset;
if (n < 3e7) {
const a = new Uint8Array(
await t.read(n, { at: e.offset })
);
return s.map((r) => {
const o = r.offset - e.offset;
return new EncodedVideoChunk({
type: r.is_sync ? "key" : "delta",
timestamp: r.cts,
duration: r.duration,
data: a.subarray(o, o + r.size)
});
});
}
return await Promise.all(
s.map(async (a) => new EncodedVideoChunk({
type: a.is_sync ? "key" : "delta",
timestamp: a.cts,
duration: a.duration,
data: await t.read(a.size, {
at: a.offset
})
}))
);
}
function Xt(s, t, e) {
const i = new OffscreenCanvas(s, t), n = i.getContext("2d");
return async (a) => (n.drawImage(a, 0, 0, s, t), a.close(), await i.convertToBlob(e));
}
function Yt(s, t) {
if (s.length === 0) return [];
let e = 0, i = 0, n = -1;
for (let c = 0; c < s.length; c++) {
const l = s[c];
if (n === -1 && t < l.cts && (n = c - 1), l.is_idr)
if (n === -1)
e = c;
else {
i = c;
break;
}
}
const a = s[n];
if (a == null) throw Error("Not found video sample by time");
const r = s.slice(0, i === 0 ? s.length : i).map((c) => ({ ...c }));
for (let c = e; c < r.length; c++) {
const l = r[c];
t < l.cts && (l.deleted = !0, l.cts = -1);
}
L(r);
const o = s.slice(a.is_idr ? n : e).map((c) => ({ ...c, cts: c.cts - t }));
for (const c of o)
c.cts < 0 && (c.deleted = !0, c.cts = -1);
return L(o), [r, o];
}
function Gt(s, t) {
if (s.length === 0) return [void 0, void 0];
if (s[0].cts >= t)
return [void 0, s.map((r) => ({ ...r }))];
if (s[s.length - 1].cts < t)
return [s.map((r) => ({ ...r })), void 0];
let i = -1;
for (let r = 0; r < s.length; r++) {
const o = s[r];
if (!(t > o.cts)) {
i = r;
break;
}
}
if (i === -1) throw Error("Not found audio sample by time");
const n = s.slice(0, i).map((r) => ({ ...r })), a = s.slice(i).map((r) => ({ ...r, cts: r.cts - t }));
return [n, a];
}
function V(s, t, e) {
if (s.state === "configured") {
for (let i = 0; i < t.length; i++) s.decode(t[i]);
s.flush().catch((i) => {
if (!(i instanceof Error)) throw i;
if (i.message.includes("Decoding error") && e.onDecodingError != null) {
e.onDecodingError(i);
return;
}
if (!i.message.includes("Aborted due to close"))
throw i;
});
}
}
function jt(s, t) {
if (t !== "avc1" && t !== "hvc1") return 0;
const e = new DataView(s.buffer);
for (let i = 0; i < s.byteLength - 4; ) {
if (t === "avc1") {
const n = e.getUint8(i + 4) & 31;
if (n === 5 || n === 7 || n === 8) return i;
} else if (t === "hvc1") {
const n = e.getUint8(i + 4) >> 1 & 63;
if (n === 19 || n === 20 || n === 32 || n === 33 || n === 34)
return i;
}
i += e.getUint32(i) + 4;
}
return -1;
}
async function Jt(s, t, e, i, n, a) {
const r = await t.createReader(), o = await st(
s.filter(
(h) => !h.deleted && h.is_sync && h.cts >= n.start && h.cts <= n.end
),
r
);
if (o.length === 0 || i.aborted) {
a(null, !0);
return;
}
let c = 0;
V(l(), o, {
onDecodingError: (h) => {
C.warn("thumbnailsByKeyFrame", h), c === 0 ? V(l(!0), o, {
onDecodingError: (u) => {
r.close(), C.error("thumbnailsByKeyFrame retry soft deocde", u);
}
}) : (a(null, !0), r.close());
}
});
function l(h = !1) {
const u = {
...e,
...h ? { hardwareAcceleration: "prefer-software" } : {}
}, d = new VideoDecoder({
output: (y) => {
c += 1;
const m = c === o.length;
a(y, m), m && (r.close(), d.state !== "closed" && d.close());
},
error: (y) => {
const m = `thumbnails decoder error: ${y.message}, config: ${JSON.stringify(u)}, state: ${JSON.stringify(
{
qSize: d.decodeQueueSize,
state: d.state,
outputCnt: c,
inputCnt: o.length
}
)}`;
throw C.error(m), Error(m);
}
});
return i.addEventListener("abort", () => {
r.close(), d.state !== "closed" && d.close();
}), d.configure(u), d;
}
}
function L(s) {
let t = 0, e = null;
for (const i of s)
if (!i.deleted) {
if (i.is_sync && (t += 1), t >= 2) break;
(e == null || i.cts < e.cts) && (e = i);
}
e != null && e.cts < 2e5 && (e.duration += e.cts, e.cts = 0);
}
function at() {
try {
const s = performance.memory;
return {
jsHeapSizeLimit: s.jsHeapSizeLimit,
totalJSHeapSize: s.totalJSHeapSize,
usedJSHeapSize: s.usedJSHeapSize,
percentUsed: (s.usedJSHeapSize / s.jsHeapSizeLimit).toFixed(3),
percentTotal: (s.totalJSHeapSize / s.jsHeapSizeLimit).toFixed(3)
};
} catch {
return {};
}
}
class R {
ready;
#t = {
// 微秒
duration: 0,
width: 0,
height: 0
};
/**
* ⚠️ 静态图片的 duration 为 Infinity
*
* 使用 Sprite 包装时需要将它的 duration 设置为有限数
*
*/
get meta() {
return { ...this.#t };
}
#n = null;
#e = [];
/**
* 静态图片可使用流、ImageBitmap 初始化
*
* 动图需要使用 VideoFrame[] 或提供图片类型
*/
constructor(t) {
const e = (i) => (this.#n = i, this.#t.width = i.width, this.#t.height = i.height, this.#t.duration = 1 / 0, { ...this.#t });
if (t instanceof ReadableStream)
this.ready = new Response(t).blob().then((i) => createImageBitmap(i)).then(e);
else if (t instanceof ImageBitmap)
this.ready = Promise.resolve(e(t));
else if (Array.isArray(t) && t.every((i) => i instanceof VideoFrame)) {
this.#e = t;
const i = this.#e[0];
if (i == null) throw Error("The frame count must be greater than 0");
this.#t = {
width: i.displayWidth,
height: i.displayHeight,
duration: this.#e.reduce(
(n, a) => n + (a.duration ?? 0),
0
)
}, this.ready = Promise.resolve({ ...this.#t, duration: 1 / 0 });
} else if ("type" in t)
this.ready = this.#s(
t.stream,
t.type
).then(() => ({
width: this.#t.width,
height: this.#t.height,
duration: 1 / 0
}));
else
throw Error("Illegal arguments");
}
async #s(t, e) {
this.#e = await Dt(t, e);
const i = this.#e[0];
if (i == null) throw Error("No frame available in gif");
this.#t = {
duration: this.#e.reduce((n, a) => n + (a.duration ?? 0), 0),
width: i.codedWidth,
height: i.codedHeight
}, C.info("ImgClip ready:", this.#t);
}
tickInterceptor = async (t, e) => e;
async tick(t) {
if (this.#n != null)
return await this.tickInterceptor(t, {
video: await createImageBitmap(this.#n),
state: "success"
});
const e = t % this.#t.duration;
return await this.tickInterceptor(t, {
video: (this.#e.find(
(i) => e >= i.timestamp && e <= i.timestamp + (i.duration ?? 0)
) ?? this.#e[0]).clone(),
state: "success"
});
}
async split(t) {
if (await this.ready, this.#n != null)
return [
new R(await createImageBitmap(this.#n)),
new R(await createImageBitmap(this.#n))
];
let e = -1;
for (let a = 0; a < this.#e.length; a++) {
const r = this.#e[a];
if (!(t > r.timestamp)) {
e = a;
break;
}
}
if (e === -1) throw Error("Not found frame by time");
const i = this.#e.slice(0, e).map((a) => new VideoFrame(a)), n = this.#e.slice(e).map(
(a) => new VideoFrame(a, {
timestamp: a.timestamp - t
})
);
return [new R(i), new R(n)];
}
async clone() {
await this.ready;
const t = this.#n == null ? this.#e.map((i) => i.clone()) : await createImageBitmap(this.#n), e = new R(t);
return e.tickInterceptor = this.tickInterceptor, e;
}
destroy() {
C.info("ImgClip destroy"), this.#n?.close(), this.#e.forEach((t) => t.close());
}
}
class A {
static ctx = null;
ready;
#t = {
// 微秒
duration: 0,
width: 0,
height: 0
};
/**
* 音频元信息
*
* ⚠️ 注意，这里是转换后（标准化）的元信息，非原始音频元信息
*/
get meta() {
return {
...this.#t,
sampleRate: x.sampleRate,
chanCount: 2
};
}
// 使用类型断言来避免 ArrayBufferLike 和 ArrayBuffer 的类型兼容性问题
#n = new Float32Array();
#e = new Float32Array();
/**
* 获取音频素材完整的 PCM 数据
*/
getPCMData() {
return [this.#n, this.#e];
}
#s;
/**
*
* @param dataSource 音频文件流
* @param opts 音频配置，控制音量、是否循环
*/
constructor(t, e = {}) {
this.#s = {
loop: !1,
volume: 1,
...e
}, this.ready = this.#a(t).then(() => ({
// audio 没有宽高，无需绘制
width: 0,
height: 0,
duration: e.loop ? 1 / 0 : this.#t.duration
}));
}
async #a(t) {
A.ctx == null && (A.ctx = new AudioContext({
sampleRate: x.sampleRate
}));
const e = performance.now(), i = t instanceof ReadableStream ? await Kt(t, A.ctx) : t;
C.info("Audio clip decoded complete:", performance.now() - e);
const n = this.#s.volume;
if (n !== 1)
for (const a of i)
for (let r = 0; r < a.length; r += 1) a[r] *= n;
this.#t.duration = i[0].length / x.sampleRate * 1e6, this.#n = i[0], this.#e = i[1] ?? this.#n, C.info(
"Audio clip convert to AudioData, time:",
performance.now() - e
);
}
/**
* 拦截 {@link AudioClip.tick} 方法返回的数据，用于对音频数据二次处理
* @param time 调用 tick 的时间
* @param tickRet tick 返回的数据
*
* @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)
*/
tickInterceptor = async (t, e) => e;
// 微秒
#r = 0;
#i = 0;
/**
* 返回上次与当前时刻差对应的音频 PCM 数据；
*
* 若差值超过 3s 或当前时间小于上次时间，则重置状态
* @example
* tick(0) // => []
* tick(1e6) // => [leftChanPCM(1s), rightChanPCM(1s)]
*
*/
async tick(t) {
if (!this.#s.loop && t >= this.#t.duration)
return await this.tickInterceptor(t, { audio: [], state: "done" });
const e = t - this.#r;
if (t < this.#r || e > 3e6)
return this.#r = t, this.#i = Math.ceil(
this.#r / 1e6 * x.sampleRate
), await this.tickInterceptor(t, {
audio: [new Float32Array(0), new Float32Array(0)],
state: "success"
});
this.#r = t;
const i = Math.ceil(
e / 1e6 * x.sampleRate
), n = this.#i + i, a = this.#s.loop ? [
z(this.#n, this.#i, n),
z(this.#e, this.#i, n)
] : [
this.#n.slice(this.#i, n),
this.#e.slice(this.#i, n)
];
return this.#i = n, await this.tickInterceptor(t, { audio: a, state: "success" });
}
/**
* 按指定时间切割，返回前后两个音频素材
* @param time 时间，单位微秒
*/
async split(t) {
await this.ready;
const e = Math.ceil(t / 1e6 * x.sampleRate), i = new A(
this.getPCMData().map((a) => a.slice(0, e)),
this.#s
), n = new A(
this.getPCMData().map((a) => a.slice(e)),
this.#s
);
return [i, n];
}
async clone() {
await this.ready;
const t = new A(this.getPCMData(), this.#s);
return await t.ready, t;
}
/**
* 销毁实例，释放资源
*/
destroy() {
this.#n = new Float32Array(0), this.#e = new Float32Array(0), C.info("---- audioclip destroy ----");
}
static concatAudioClip = Qt;
}
async function Qt(s, t) {
const e = [];
for (const i of s)
await i.ready, e.push(i.getPCMData());
return new A(tt(e), t);
}
async function Kt(s, t) {
const e = await new Response(s).arrayBuffer();
return $(await t.decodeAudioData(e));
}
class rt {
static ctx = null;
ready;
#t = {
// 微秒
duration: 0,
width: 0,
height: 0
};
get meta() {
return {
...this.#t
};
}
#n = () => {
};
/**
* 实时流的音轨
*/
audioTrack;
#e = null;
#s;
constructor(t) {
this.#s = t, this.audioTrack = t.getAudioTracks()[0] ?? null, this.#t.duration = 1 / 0;
const e = t.getVideoTracks()[0];
e != null ? (e.contentHint = "motion", this.ready = new Promise((i) => {
this.#n = qt(e, (n) => {
this.#t.width = n.width, this.#t.height = n.height, this.#e = n, i(this.meta);
});
})) : this.ready = Promise.resolve(this.meta);
}
async tick() {
return {
video: this.#e == null ? null : await createImageBitmap(this.#e),
audio: [],
state: "success"
};
}
async split() {
return [await this.clone(), await this.clone()];
}
async clone() {
return new rt(this.#s.clone());
}
destroy() {
this.#s.getTracks().forEach((t) => t.stop()), this.#n();
}
}
function qt(s, t) {
let e = !1, i;
return U(
new MediaStreamTrackProcessor({
track: s
}).readable,
{
onChunk: async (n) => {
if (!e) {
const { displayHeight: a, displayWidth: r } = n, o = r ?? 0, c = a ?? 0, l = new OffscreenCanvas(o, c);
i = l.getContext("2d"), t(l), e = !0;
}
i.drawImage(n, 0, 0), n.close();
},
onDone: async () => {
}
}
);
}
class E {
ready;
#t = [];
#n = {
width: 0,
height: 0,
duration: 0
};
get meta() {
return { ...this.#n };
}
#e = {
color: "#FFF",
textBgColor: null,
type: "srt",
fontSize: 30,
letterSpacing: null,
bottomOffset: 30,
fontFamily: "Noto Sans SC",
strokeStyle: "#000",
lineWidth: null,
lineCap: null,
lineJoin: null,
textShadow: {
offsetX: 2,
offsetY: 2,
blur: 4,
color: "#000"
},
videoWidth: 1280,
videoHeight: 720,
fontWeight: "normal",
fontStyle: "normal"
};
#s;
#a;
#r = null;
#i = 0;
#o = 0;
constructor(t, e) {
if (this.#t = Array.isArray(t) ? t : Zt(t).map(({ start: h, end: u, text: d }) => ({
start: h * 1e6,
end: u * 1e6,
text: d
})), this.#t.length === 0) throw Error("No subtitles content");
this.#e = Object.assign(this.#e, e), this.#o = e.textBgColor == null ? 0 : (e.fontSize ?? 50) * 0.2;
const {
fontSize: i,
fontFamily: n,
fontWeight: a,
fontStyle: r,
videoWidth: o,
videoHeight: c,
letterSpacing: l
} = this.#e;
this.#i = i + this.#o * 2, this.#s = new OffscreenCanvas(o, c), this.#a = this.#s.getContext("2d"), this.#a.font = `${r} ${a} ${i}px ${n}`, this.#a.textAlign = "center", this.#a.textBaseline = "top", this.#a.letterSpacing = l ?? "0px", this.#n = {