@fly-cut/av-cliper

Version:

WebCodecs-based, combine video, audio, images, text, with animation support 基于 WebCodecs 合成视频、音频、图片、文字，支持动画

1 lines • 276 kB

Source Map (JSON)

{"version":3,"file":"av-cliper.umd.cjs","sources":["../src/dom-utils.ts","../src/av-utils.ts","../src/clips/iclip.ts","../src/mp4-utils/mp4box-utils.ts","../src/clips/mp4-clip.ts","../src/clips/img-clip.ts","../src/clips/audio-clip.ts","../src/clips/media-stream-clip.ts","../src/clips/embed-subtitles-clip.ts","../src/clips/text-clip.ts","../src/mp4-utils/sample-transform.ts","../src/mp4-utils/index.ts","../src/chromakey.ts","../src/sprite/rect.ts","../src/sprite/base-sprite.ts","../src/sprite/offscreen-sprite.ts","../src/sprite/visible-sprite.ts","../src/combinator.ts"],"sourcesContent":["// 在主线程中执行的工具函数\n\n/**\n * 创建一个新的 HTML 元素\n * @param tagName - 要创建的元素的标签名\n * @returns 新创建的 HTML 元素\n */\nexport function createEl(tagName: string): HTMLElement {\n return document.createElement(tagName);\n}\n\n/**\n * 将文本渲染为图片\n * @param txt - 要渲染的文本\n * @param cssText - 应用于文本的 CSS 样式\n * @returns 渲染后的图片元素\n */\nexport function renderTxt2Img(txt: string, cssText: string): HTMLImageElement {\n const div = createEl('pre');\n div.style.cssText = `margin: 0; ${cssText}; visibility: hidden; position: fixed;`;\n div.textContent = txt;\n document.body.appendChild(div);\n\n const { width, height } = div.getBoundingClientRect();\n // 计算出 rect，立即从dom移除\n div.remove();\n div.style.visibility = 'visible';\n\n const img = new Image();\n img.width = width;\n img.height = height;\n const svgStr = `\n <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"${width}\" height=\"${height}\">\n <foreignObject width=\"100%\" height=\"100%\">\n <div xmlns=\"http://www.w3.org/1999/xhtml\">${div.outerHTML}</div>\n </foreignObject>\n </svg>\n `\n .replace(/\\t/g, '')\n .replace(/#/g, '%23');\n\n img.src = `data:image/svg+xml;charset=utf-8,${svgStr}`;\n return img;\n}\n\n/**\n * 将文本渲染为 {@link ImageBitmap}，用来创建 {@link ImgClip}\n * @param txt - 要渲染的文本\n * @param cssText - 应用于文本的 CSS 样式\n *\n * @example\n * new ImgClip(\n * await renderTxt2ImgBitmap(\n * '水印',\n * `font-size:40px; color: white; text-shadow: 2px 2px 6px red;`,\n * )\n * )\n */\nexport async function renderTxt2ImgBitmap(\n txt: string,\n cssText: string,\n): Promise<ImageBitmap> {\n const imgEl = renderTxt2Img(txt, cssText);\n await new Promise((resolve) => {\n imgEl.onload = resolve;\n });\n const cvs = new OffscreenCanvas(imgEl.width, imgEl.height);\n const ctx = cvs.getContext('2d');\n ctx?.drawImage(imgEl, 0, 0, imgEl.width, imgEl.height);\n return await createImageBitmap(cvs);\n}\n","// 能同时在 worker 和主线程中运行的工具函数\n\nimport { workerTimer } from '@fly-cut/internal-utils';\nimport * as waveResampler from 'wave-resampler';\n\n/**\n * 合并（串联）多个 Float32Array，通常用于合并 PCM 数据\n */\nexport function concatFloat32Array(bufs: Float32Array[]): Float32Array {\n const rs = new Float32Array(\n bufs.map((buf) => buf.length).reduce((a, b) => a + b),\n );\n\n let offset = 0;\n for (const buf of bufs) {\n rs.set(buf, offset);\n offset += buf.length;\n }\n\n return rs;\n}\n\n/**\n * 将小片段的 PCM 合并成一个大片段\n * @param fragments 小片段 PCM，子元素是不同声道的原始 PCM 数据\n */\nexport function concatPCMFragments(\n fragments: Float32Array[][],\n): Float32Array[] {\n // fragments: [[chan0, chan1], [chan0, chan1]...]\n // chanListPCM: [[chan0, chan0...], [chan1, chan1...]]\n const chanListPCM: Float32Array[][] = [];\n for (let i = 0; i < fragments.length; i += 1) {\n for (let j = 0; j < fragments[i].length; j += 1) {\n if (chanListPCM[j] == null) chanListPCM[j] = [];\n chanListPCM[j].push(fragments[i][j]);\n }\n }\n // [bigChan0, bigChan1]\n return chanListPCM.map(concatFloat32Array);\n}\n\n/**\n * 从 AudioData 中提取 PCM 数据的工具函数\n */\nexport function extractPCM4AudioData(ad: AudioData): Float32Array[] {\n if (ad.format === 'f32-planar') {\n const rs = [];\n for (let idx = 0; idx < ad.numberOfChannels; idx += 1) {\n const chanBufSize = ad.allocationSize({ planeIndex: idx });\n const chanBuf = new ArrayBuffer(chanBufSize);\n ad.copyTo(chanBuf, { planeIndex: idx });\n rs.push(new Float32Array(chanBuf));\n }\n return rs;\n } else if (ad.format === 'f32') {\n const buf = new ArrayBuffer(ad.allocationSize({ planeIndex: 0 }));\n ad.copyTo(buf, { planeIndex: 0 });\n return convertF32ToPlanar(new Float32Array(buf), ad.numberOfChannels);\n } else if (ad.format === 's16') {\n const buf = new ArrayBuffer(ad.allocationSize({ planeIndex: 0 }));\n ad.copyTo(buf, { planeIndex: 0 });\n return convertS16ToF32Planar(new Int16Array(buf), ad.numberOfChannels);\n }\n throw Error('Unsupported audio data format');\n}\n\n/**\n * Convert s16 PCM to f32-planar\n * @param pcmS16Data - The s16 PCM data.\n * @param numChannels - Number of audio channels.\n * @returns An array of Float32Array, each containing the audio data for one channel.\n */\nfunction convertS16ToF32Planar(pcmS16Data: Int16Array, numChannels: number) {\n const numSamples = pcmS16Data.length / numChannels;\n const planarData = Array.from(\n { length: numChannels },\n () => new Float32Array(numSamples),\n );\n\n for (let i = 0; i < numSamples; i++) {\n for (let channel = 0; channel < numChannels; channel++) {\n const sample = pcmS16Data[i * numChannels + channel];\n planarData[channel][i] = sample / 32768; // Normalize to range [-1.0, 1.0]\n }\n }\n\n return planarData;\n}\n\nfunction convertF32ToPlanar(pcmF32Data: Float32Array, numChannels: number) {\n const numSamples = pcmF32Data.length / numChannels;\n const planarData = Array.from(\n { length: numChannels },\n () => new Float32Array(numSamples),\n );\n\n for (let i = 0; i < numSamples; i++) {\n for (let channel = 0; channel < numChannels; channel++) {\n planarData[channel][i] = pcmF32Data[i * numChannels + channel];\n }\n }\n\n return planarData;\n}\n\n/**\n * 从 AudioBuffer 中提取 PCM\n */\nexport function extractPCM4AudioBuffer(ab: AudioBuffer): Float32Array[] {\n return Array(ab.numberOfChannels)\n .fill(0)\n .map((_, idx) => {\n return ab.getChannelData(idx);\n });\n}\n\n/**\n * 调整音频数据的音量\n * @param ad - 要调整的音频对象\n * @param volume - 音量调整系数（0.0 - 1.0）\n * @returns 调整音量后的新音频数据\n */\nexport function adjustAudioDataVolume(ad: AudioData, volume: number) {\n const data = new Float32Array(\n concatFloat32Array(extractPCM4AudioData(ad)),\n ).map((v) => v * volume);\n const newAd = new AudioData({\n sampleRate: ad.sampleRate,\n numberOfChannels: ad.numberOfChannels,\n timestamp: ad.timestamp,\n format: ad.format,\n numberOfFrames: ad.numberOfFrames,\n data,\n });\n ad.close();\n return newAd;\n}\n\n/**\n * 解码图像流，返回一个视频帧数组。\n *\n * @param stream - 包含图像数据的可读流。\n * @param type - 图像的 MIME 类型，例如 'image/jpeg'。\n *\n * @returns 返回一个 Promise，该 Promise 在解码完成后解析为 {@link VideoFrame} 数组。\n *\n * @see [解码动图](https://webav-tech.github.io/WebAV/demo/1_3-decode-image)\n *\n * @example\n *\n * const frames = await decodeImg(\n * (await fetch('<gif url>')).body!,\n * `image/gif`,\n * );\n */\nexport async function decodeImg(\n stream: ReadableStream<Uint8Array>,\n type: string,\n): Promise<VideoFrame[]> {\n const init = {\n type,\n data: stream,\n };\n const imageDecoder = new ImageDecoder(init);\n\n await Promise.all([imageDecoder.completed, imageDecoder.tracks.ready]);\n\n let frameCnt = imageDecoder.tracks.selectedTrack?.frameCount ?? 1;\n\n const rs: VideoFrame[] = [];\n for (let i = 0; i < frameCnt; i += 1) {\n rs.push((await imageDecoder.decode({ frameIndex: i })).image);\n }\n return rs;\n}\n\n/**\n * 混合双通道音轨的 PCM 数据，并将多声道并排成一个 Float32Array 输出\n * @param audios - 一个二维数组，每个元素是一个 Float32Array 数组，代表一个音频流的 PCM 数据。\n * 每个 Float32Array 数组的第一个元素是左声道数据，第二个元素（如果有）是右声道数据。\n * 如果只有左声道数据，则右声道将复用左声道数据。\n *\n * @returns 返回一个 Float32Array，返回结果是将这个一个音轨的左右声道并排成 Float32Array。\n *\n * @example\n *\n * const audios = [\n * [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.4, 0.5, 0.6])],\n * [new Float32Array([0.7, 0.8, 0.9])],\n * ];\n * const mixed = mixinPCM(audios);\n */\nexport function mixinPCM(audios: Float32Array[][]): Float32Array {\n const maxLen = Math.max(...audios.map((a) => a[0]?.length ?? 0));\n const data = new Float32Array(maxLen * 2);\n\n for (let bufIdx = 0; bufIdx < maxLen; bufIdx++) {\n let chan0 = 0;\n let chan1 = 0;\n for (let trackIdx = 0; trackIdx < audios.length; trackIdx++) {\n const _c0 = audios[trackIdx][0]?.[bufIdx] ?? 0;\n // 如果是单声道 PCM，第二声道复用第一声道数据\n const _c1 = audios[trackIdx][1]?.[bufIdx] ?? _c0;\n chan0 += _c0;\n chan1 += _c1;\n }\n data[bufIdx] = chan0;\n data[bufIdx + maxLen] = chan1;\n }\n\n return data;\n}\n\n/**\n * 对 PCM 音频数据进行重采样。\n *\n * @param pcmData - 一个 Float32Array 数组，每个元素代表一个声道的 PCM 数据。\n * @param curRate - 当前的采样率。\n * @param target - 目标参数对象。\n * @param target.rate - 目标采样率。\n * @param target.chanCount - 目标声道数。\n *\n * @returns 返回一个 Promise，该 Promise 在重采样完成后解析为一个 Float32Array 数组，每个元素代表一个声道的 PCM 数据。\n *\n * @example\n *\n * const pcmData = [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.4, 0.5, 0.6])];\n * const curRate = 44100;\n * const target = { rate: 48000, chanCount: 2 };\n * const resampled = await audioResample(pcmData, curRate, target);\n */\nexport async function audioResample(\n pcmData: Float32Array[],\n curRate: number,\n target: {\n rate: number;\n chanCount: number;\n },\n): Promise<Float32Array[]> {\n const chanCnt = pcmData.length;\n const emptyPCM = Array(target.chanCount)\n .fill(0)\n .map(() => new Float32Array(0));\n if (chanCnt === 0) return emptyPCM;\n\n const len = Math.max(...pcmData.map((c) => c.length));\n if (len === 0) return emptyPCM;\n\n // The Worker scope does not have access to OfflineAudioContext\n if (globalThis.OfflineAudioContext == null) {\n return pcmData.map(\n (p) =>\n new Float32Array(\n waveResampler.resample(p, curRate, target.rate, {\n method: 'sinc',\n LPF: false,\n }),\n ),\n );\n }\n\n const ctx = new globalThis.OfflineAudioContext(\n target.chanCount,\n (len * target.rate) / curRate,\n target.rate,\n );\n const abSource = ctx.createBufferSource();\n const ab = ctx.createBuffer(chanCnt, len, curRate);\n pcmData.forEach((d, idx) => ab.copyToChannel(d, idx));\n\n abSource.buffer = ab;\n abSource.connect(ctx.destination);\n abSource.start();\n\n return extractPCM4AudioBuffer(await ctx.startRendering());\n}\n\n/**\n * 使当前执行环境暂停一段时间。\n * @param time - 暂停的时间，单位为毫秒。\n * @example\n * await sleep(1000); // 暂停 1 秒\n */\nexport function sleep(time: number): Promise<void> {\n return new Promise((resolve) => {\n const stop = workerTimer(() => {\n stop();\n resolve();\n }, time);\n });\n}\n\n/**\n * 从给定的 Float32Array 中提取一个环形切片，超出边界从 0 开始循环\n *\n * 主要用于截取 PCM 实现循环播放\n *\n * @param data - 输入的 Float32Array。\n * @param start - 切片的开始索引。\n * @param end - 切片的结束索引。\n * @returns - 返回一个新的 Float32Array，包含从 start 到 end 的数据。\n *\n * @example\n * const data = new Float32Array([0, 1, 2, 3, 4, 5]);\n * ringSliceFloat32Array(data, 4, 6); // => Float32Array [4, 5, 0]\n */\nexport function ringSliceFloat32Array(\n data: Float32Array,\n start: number,\n end: number,\n): Float32Array {\n const cnt = end - start;\n const rs = new Float32Array(cnt);\n let i = 0;\n while (i < cnt) {\n rs[i] = data[(start + i) % data.length];\n i += 1;\n }\n return rs;\n}\n\n/**\n * 函数节流\n */\nexport function throttle<F extends (...args: any[]) => any>(\n func: F,\n wait: number,\n): (...rest: Parameters<F>) => undefined | ReturnType<F> {\n let lastTime: number;\n return function (this: any, ...rest) {\n if (lastTime == null || performance.now() - lastTime > wait) {\n lastTime = performance.now();\n return func.apply(this, rest);\n }\n };\n}\n\n/**\n * 改变 PCM 数据的播放速率，1 表示正常播放，0.5 表示播放速率减半，2 表示播放速率加倍\n */\nexport function changePCMPlaybackRate(\n pcmData: Float32Array,\n playbackRate: number,\n) {\n // 计算新的采样率\n const newLength = Math.floor(pcmData.length / playbackRate);\n const newPcmData = new Float32Array(newLength);\n\n // 线性插值\n for (let i = 0; i < newLength; i++) {\n // 原始数据中的位置\n const originalIndex = i * playbackRate;\n const intIndex = Math.floor(originalIndex);\n const frac = originalIndex - intIndex;\n\n // 边界检查\n if (intIndex + 1 < pcmData.length) {\n newPcmData[i] =\n pcmData[intIndex] * (1 - frac) + pcmData[intIndex + 1] * frac;\n } else {\n newPcmData[i] = pcmData[intIndex]; // 最后一个样本\n }\n }\n\n return newPcmData;\n}\n","interface IClipMeta {\n width: number;\n height: number;\n duration: number;\n}\n\n/**\n * 所有素材需要实现的接口\n *\n * 素材（Clip）是不同数据类型的抽象，给其他模块提供数据\n *\n * WebAV 内置了 {@link MP4Clip}, {@link AudioClip}, {@link ImgClip}, {@link MediaStreamClip} 等常用素材，用于给 {@link Combinator} {@link AVCanvas} 提供数据\n *\n * 你只需实现该接口即可自定义素材，拥有最大的灵活度来生成视频内容，比如动画、转场效果等\n * @see [自定义素材](https://webav-tech.github.io/WebAV/demo/2_6-custom-clip)\n *\n */\nexport interface IClip {\n /**\n * 从素材中提取指定时间数据\n * @param time 时间，单位微秒\n */\n tick: (time: number) => Promise<{\n video?: VideoFrame | ImageBitmap | null;\n audio?: Float32Array[];\n state: 'done' | 'success';\n }>;\n\n /**\n * 当素材准备完成，ready 会切换到 resolved 状态\n */\n readonly ready: Promise<IClipMeta>;\n\n /**\n * 数据元数据\n */\n readonly meta: IClipMeta;\n\n /**\n * clone，返回一个新素材\n */\n clone: () => Promise<this>;\n\n /**\n * 按指定时间切割，返回该时刻前后两个新素材，常用于剪辑场景按时间分割素材\n *\n * 该方法不会破坏原素材的数据\n *\n * @param time 时间，微秒\n * @returns\n */\n split?: (time: number) => Promise<[this, this]>;\n\n /**\n * 销毁实例，释放资源\n */\n destroy: () => void;\n}\n\n/**\n * 默认的音频设置，⚠️ 不要变更它的值 ⚠️\n */\nexport const DEFAULT_AUDIO_CONF = {\n sampleRate: 48000,\n channelCount: 2,\n codec: 'mp4a.40.2',\n} as const;\n","import mp4box, {\n AudioTrackOpts,\n ESDSBoxParser,\n MP4ABoxParser,\n MP4ArrayBuffer,\n MP4File,\n MP4Info,\n MP4Sample,\n TrakBoxParser,\n VideoTrackOpts,\n} from '@webav/mp4box.js';\nimport { DEFAULT_AUDIO_CONF } from '../clips';\nimport { file } from 'opfs-tools';\n\nexport function extractFileConfig(file: MP4File, info: MP4Info) {\n const vTrack = info.videoTracks[0];\n const rs: {\n videoTrackConf?: VideoTrackOpts;\n videoDecoderConf?: Parameters<VideoDecoder['configure']>[0];\n audioTrackConf?: AudioTrackOpts;\n audioDecoderConf?: Parameters<AudioDecoder['configure']>[0];\n } = {};\n if (vTrack != null) {\n const videoDesc = parseVideoCodecDesc(file.getTrackById(vTrack.id)).buffer;\n const { descKey, type } = vTrack.codec.startsWith('avc1')\n ? { descKey: 'avcDecoderConfigRecord', type: 'avc1' }\n : vTrack.codec.startsWith('hvc1')\n ? { descKey: 'hevcDecoderConfigRecord', type: 'hvc1' }\n : { descKey: '', type: '' };\n if (descKey !== '') {\n rs.videoTrackConf = {\n timescale: vTrack.timescale,\n duration: vTrack.duration,\n width: vTrack.video.width,\n height: vTrack.video.height,\n brands: info.brands,\n type,\n [descKey]: videoDesc,\n };\n }\n\n rs.videoDecoderConf = {\n codec: vTrack.codec,\n codedHeight: vTrack.video.height,\n codedWidth: vTrack.video.width,\n description: videoDesc,\n };\n }\n\n const aTrack = info.audioTracks[0];\n if (aTrack != null) {\n const esdsBox = getESDSBoxFromMP4File(file);\n rs.audioTrackConf = {\n timescale: aTrack.timescale,\n samplerate: aTrack.audio.sample_rate,\n channel_count: aTrack.audio.channel_count,\n hdlr: 'soun',\n type: aTrack.codec.startsWith('mp4a') ? 'mp4a' : aTrack.codec,\n description: getESDSBoxFromMP4File(file),\n };\n rs.audioDecoderConf = {\n codec: aTrack.codec.startsWith('mp4a')\n ? DEFAULT_AUDIO_CONF.codec\n : aTrack.codec,\n numberOfChannels: aTrack.audio.channel_count,\n sampleRate: aTrack.audio.sample_rate,\n ...(esdsBox == null ? {} : parseAudioInfo4ESDSBox(esdsBox)),\n };\n }\n return rs;\n}\n\n// track is H.264, H.265 or VPX.\nfunction parseVideoCodecDesc(track: TrakBoxParser): Uint8Array {\n for (const entry of track.mdia.minf.stbl.stsd.entries) {\n // @ts-expect-error\n const box = entry.avcC ?? entry.hvcC ?? entry.av1C ?? entry.vpcC;\n if (box != null) {\n const stream = new mp4box.DataStream(\n undefined,\n 0,\n mp4box.DataStream.BIG_ENDIAN,\n );\n box.write(stream);\n return new Uint8Array(stream.buffer.slice(8)); // Remove the box header.\n }\n }\n throw Error('avcC, hvcC, av1C or VPX not found');\n}\n\nfunction getESDSBoxFromMP4File(file: MP4File, codec = 'mp4a') {\n const mp4aBox = file.moov?.traks\n .map((t) => t.mdia.minf.stbl.stsd.entries)\n .flat()\n .find(({ type }) => type === codec) as MP4ABoxParser;\n\n return mp4aBox?.esds;\n}\n\n// 解决封装层音频信息标识错误，导致解码异常\nfunction parseAudioInfo4ESDSBox(esds: ESDSBoxParser) {\n const decoderConf = esds.esd.descs[0]?.descs[0];\n if (decoderConf == null) return {};\n\n const [byte1, byte2] = decoderConf.data;\n // sampleRate 是第一字节后 3bit + 第二字节前 1bit\n const sampleRateIdx = ((byte1 & 0x07) << 1) + (byte2 >> 7);\n // numberOfChannels 是第二字节 [2, 5] 4bit\n const numberOfChannels = (byte2 & 0x7f) >> 3;\n const sampleRateEnum = [\n 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025,\n 8000, 7350,\n ] as const;\n return {\n sampleRate: sampleRateEnum[sampleRateIdx],\n numberOfChannels,\n };\n}\n\n/**\n * 快速解析 mp4 文件，如果是非 fMP4 格式，会优先解析 moov box（略过 mdat）避免占用过多内存\n */\nexport async function quickParseMP4File(\n reader: Awaited<ReturnType<ReturnType<typeof file>['createReader']>>,\n onReady: (data: { mp4boxFile: MP4File; info: MP4Info }) => void,\n onSamples: (\n id: number,\n sampleType: 'video' | 'audio',\n samples: MP4Sample[],\n ) => void,\n) {\n const mp4boxFile = mp4box.createFile(false);\n mp4boxFile.onReady = (info) => {\n onReady({ mp4boxFile, info });\n const vTrackId = info.videoTracks[0]?.id;\n if (vTrackId != null)\n mp4boxFile.setExtractionOptions(vTrackId, 'video', { nbSamples: 100 });\n\n const aTrackId = info.audioTracks[0]?.id;\n if (aTrackId != null)\n mp4boxFile.setExtractionOptions(aTrackId, 'audio', { nbSamples: 100 });\n\n mp4boxFile.start();\n };\n mp4boxFile.onSamples = onSamples;\n\n await parse();\n\n async function parse() {\n let cursor = 0;\n const maxReadSize = 30 * 1024 * 1024;\n while (true) {\n const data = (await reader.read(maxReadSize, {\n at: cursor,\n })) as MP4ArrayBuffer;\n if (data.byteLength === 0) break;\n data.fileStart = cursor;\n const nextPos = mp4boxFile.appendBuffer(data);\n if (nextPos == null) break;\n cursor = nextPos;\n }\n\n mp4boxFile.stop();\n }\n}\n","import { MP4Info, MP4Sample } from '@webav/mp4box.js';\nimport { audioResample, extractPCM4AudioData, sleep } from '../av-utils';\nimport { Log } from '@fly-cut/internal-utils';\nimport {\n extractFileConfig,\n quickParseMP4File,\n} from '../mp4-utils/mp4box-utils';\nimport { DEFAULT_AUDIO_CONF, IClip } from './iclip';\nimport { file, tmpfile, write } from 'opfs-tools';\n\nlet CLIP_ID = 0;\n\ntype OPFSToolFile = ReturnType<typeof file>;\nfunction isOTFile(obj: any): obj is OPFSToolFile {\n return obj.kind === 'file' && obj.createReader instanceof Function;\n}\n\n// 用于内部创建 MP4Clip 实例\ntype MPClipCloneArgs = Awaited<ReturnType<typeof mp4FileToSamples>> & {\n localFile: OPFSToolFile;\n};\n\ninterface MP4DecoderConf {\n video: VideoDecoderConfig | null;\n audio: AudioDecoderConfig | null;\n}\n\ninterface MP4ClipOpts {\n audio?: boolean | { volume: number };\n /**\n * 不安全，随时可能废弃\n */\n __unsafe_hardwareAcceleration__?: HardwarePreference;\n}\n\ntype ExtMP4Sample = Omit<MP4Sample, 'data'> & {\n is_idr: boolean;\n deleted?: boolean;\n data: null | Uint8Array;\n};\n\ntype LocalFileReader = Awaited<ReturnType<OPFSToolFile['createReader']>>;\n\ntype ThumbnailOpts = {\n start: number;\n end: number;\n step: number;\n};\n\n/**\n * MP4 素材，解析 MP4 文件，使用 {@link MP4Clip.tick} 按需解码指定时间的图像帧\n *\n * 可用于实现视频抽帧、生成缩略图、视频编辑等功能\n *\n * @example\n * new MP4Clip((await fetch('<mp4 url>')).body)\n * new MP4Clip(mp4File.stream())\n *\n * @see {@link Combinator}\n * @see [AVCanvas](../../av-canvas/classes/AVCanvas.html)\n *\n * @see [解码播放视频](https://webav-tech.github.io/WebAV/demo/1_1-decode-video)\n */\nexport class MP4Clip implements IClip {\n #insId = CLIP_ID++;\n\n #log = Log.create(`MP4Clip id:${this.#insId},`);\n\n ready: IClip['ready'];\n\n #destroyed = false;\n\n #meta = {\n // 微秒\n duration: 0,\n width: 0,\n height: 0,\n audioSampleRate: 0,\n audioChanCount: 0,\n };\n\n get meta() {\n return { ...this.#meta };\n }\n\n #localFile: OPFSToolFile;\n\n #headerBoxPos: Array<{ start: number; size: number }> = [];\n /**\n * 提供视频头（box: ftyp, moov）的二进制数据\n * 使用任意 mp4 demxer 解析即可获得详细的视频信息\n * 单元测试包含使用 mp4box.js 解析示例代码\n */\n async getFileHeaderBinData() {\n await this.ready;\n const oFile = await this.#localFile.getOriginFile();\n if (oFile == null) throw Error('MP4Clip localFile is not origin file');\n\n return await new Blob(\n this.#headerBoxPos.map(({ start, size }) =>\n oFile.slice(start, start + size),\n ),\n ).arrayBuffer();\n }\n\n #volume = 1;\n\n #videoSamples: ExtMP4Sample[] = [];\n\n #audioSamples: ExtMP4Sample[] = [];\n\n #videoFrameFinder: VideoFrameFinder | null = null;\n #audioFrameFinder: AudioFrameFinder | null = null;\n\n #decoderConf: {\n video: VideoDecoderConfig | null;\n audio: AudioDecoderConfig | null;\n } = {\n video: null,\n audio: null,\n };\n\n #opts: MP4ClipOpts = { audio: true };\n\n constructor(\n source: OPFSToolFile | ReadableStream<Uint8Array> | MPClipCloneArgs,\n opts: MP4ClipOpts = {},\n ) {\n if (\n !(source instanceof ReadableStream) &&\n !isOTFile(source) &&\n !Array.isArray(source.videoSamples)\n ) {\n throw Error('Illegal argument');\n }\n\n this.#opts = { audio: true, ...opts };\n this.#volume =\n typeof opts.audio === 'object' && 'volume' in opts.audio\n ? opts.audio.volume\n : 1;\n\n const initByStream = async (s: ReadableStream) => {\n await write(this.#localFile, s);\n return this.#localFile;\n };\n\n this.#localFile = isOTFile(source)\n ? source\n : 'localFile' in source\n ? source.localFile // from clone\n : tmpfile();\n\n this.ready = (\n source instanceof ReadableStream\n ? initByStream(source).then((otFile) =>\n mp4FileToSamples(otFile, this.#opts),\n )\n : isOTFile(source)\n ? mp4FileToSamples(source, this.#opts)\n : Promise.resolve(source)\n ).then(\n async ({ videoSamples, audioSamples, decoderConf, headerBoxPos }) => {\n this.#videoSamples = videoSamples;\n this.#audioSamples = audioSamples;\n this.#decoderConf = decoderConf;\n this.#headerBoxPos = headerBoxPos;\n\n const { videoFrameFinder, audioFrameFinder } = genDecoder(\n {\n video:\n decoderConf.video == null\n ? null\n : {\n ...decoderConf.video,\n hardwareAcceleration:\n this.#opts.__unsafe_hardwareAcceleration__,\n },\n audio: decoderConf.audio,\n },\n await this.#localFile.createReader(),\n videoSamples,\n audioSamples,\n this.#opts.audio !== false ? this.#volume : 0,\n );\n this.#videoFrameFinder = videoFrameFinder;\n this.#audioFrameFinder = audioFrameFinder;\n\n this.#meta = genMeta(decoderConf, videoSamples, audioSamples);\n this.#log.info('MP4Clip meta:', this.#meta);\n return { ...this.#meta };\n },\n );\n }\n\n /**\n * 拦截 {@link MP4Clip.tick} 方法返回的数据，用于对图像、音频数据二次处理\n * @param time 调用 tick 的时间\n * @param tickRet tick 返回的数据\n *\n * @see [移除视频绿幕背景](https://webav-tech.github.io/WebAV/demo/3_2-chromakey-video)\n */\n tickInterceptor: <T extends Awaited<ReturnType<MP4Clip['tick']>>>(\n time: number,\n tickRet: T,\n ) => Promise<T> = async (_, tickRet) => tickRet;\n\n /**\n * 获取素材指定时刻的图像帧、音频数据\n * @param time 微秒\n */\n async tick(time: number): Promise<{\n video?: VideoFrame;\n audio: Float32Array[];\n state: 'success' | 'done';\n }> {\n if (time >= this.#meta.duration) {\n return await this.tickInterceptor(time, {\n audio: (await this.#audioFrameFinder?.find(time)) ?? [],\n state: 'done',\n });\n }\n\n const [audio, video] = await Promise.all([\n this.#audioFrameFinder?.find(time) ?? [],\n this.#videoFrameFinder?.find(time),\n ]);\n\n if (video == null) {\n return await this.tickInterceptor(time, {\n audio,\n state: 'success',\n });\n }\n\n return await this.tickInterceptor(time, {\n video,\n audio,\n state: 'success',\n });\n }\n\n #thumbAborter = new AbortController();\n /**\n * 生成缩略图，默认每个关键帧生成一个 100px 宽度的缩略图。\n *\n * @param imgWidth 缩略图宽度，默认 100\n * @param opts Partial<ThumbnailOpts>\n * @returns Promise<Array<{ ts: number; img: Blob }>>\n */\n async thumbnails(\n imgWidth = 100,\n opts?: Partial<ThumbnailOpts>,\n ): Promise<Array<{ ts: number; img: Blob }>> {\n this.#thumbAborter.abort();\n this.#thumbAborter = new AbortController();\n const aborterSignal = this.#thumbAborter.signal;\n\n await this.ready;\n const abortMsg = 'generate thumbnails aborted';\n if (aborterSignal.aborted) throw Error(abortMsg);\n\n const { width, height } = this.#meta;\n const convtr = createVF2BlobConvtr(\n imgWidth,\n Math.round(height * (imgWidth / width)),\n { quality: 0.1, type: 'image/png' },\n );\n\n return new Promise<Array<{ ts: number; img: Blob }>>(\n async (resolve, reject) => {\n let pngPromises: Array<{ ts: number; img: Promise<Blob> }> = [];\n const vc = this.#decoderConf.video;\n if (vc == null || this.#videoSamples.length === 0) {\n resolver();\n return;\n }\n aborterSignal.addEventListener('abort', () => {\n reject(Error(abortMsg));\n });\n\n async function resolver() {\n if (aborterSignal.aborted) return;\n resolve(\n await Promise.all(\n pngPromises.map(async (it) => ({\n ts: it.ts,\n img: await it.img,\n })),\n ),\n );\n }\n\n function pushPngPromise(vf: VideoFrame) {\n pngPromises.push({\n ts: vf.timestamp,\n img: convtr(vf),\n });\n }\n\n const { start = 0, end = this.#meta.duration, step } = opts ?? {};\n if (step) {\n let cur = start;\n // 创建一个新的 VideoFrameFinder 实例，避免与 tick 方法共用而导致冲突\n const videoFrameFinder = new VideoFrameFinder(\n await this.#localFile.createReader(),\n this.#videoSamples,\n {\n ...vc,\n hardwareAcceleration: this.#opts.__unsafe_hardwareAcceleration__,\n },\n );\n while (cur <= end && !aborterSignal.aborted) {\n const vf = await videoFrameFinder.find(cur);\n if (vf) pushPngPromise(vf);\n cur += step;\n }\n videoFrameFinder.destroy();\n resolver();\n } else {\n await thumbnailByKeyFrame(\n this.#videoSamples,\n this.#localFile,\n vc,\n aborterSignal,\n { start, end },\n (vf, done) => {\n if (vf != null) pushPngPromise(vf);\n if (done) resolver();\n },\n );\n }\n },\n );\n }\n\n async split(time: number) {\n await this.ready;\n\n if (time <= 0 || time >= this.#meta.duration)\n throw Error('\"time\" out of bounds');\n\n const [preVideoSlice, postVideoSlice] = splitVideoSampleByTime(\n this.#videoSamples,\n time,\n );\n const [preAudioSlice, postAudioSlice] = splitAudioSampleByTime(\n this.#audioSamples,\n time,\n );\n const preClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: preVideoSlice ?? [],\n audioSamples: preAudioSlice ?? [],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n const postClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: postVideoSlice ?? [],\n audioSamples: postAudioSlice ?? [],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await Promise.all([preClip.ready, postClip.ready]);\n\n return [preClip, postClip] as [this, this];\n }\n\n async removeSegment(startTime: number, endTime: number): Promise<MP4Clip> {\n await this.ready;\n\n if (\n startTime < 0 ||\n endTime > this.#meta.duration ||\n startTime >= endTime\n ) {\n throw Error('Invalid time range');\n }\n\n // 处理视频和音频样本，删除指定时间段\n const newVideoSamples = this.#removeVideoSamples(startTime, endTime);\n const newAudioSamples = this.#removeAudioSamples(startTime, endTime);\n\n // 创建新的 MP4Clip 实例\n const newClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: newVideoSamples,\n audioSamples: newAudioSamples,\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n\n await newClip.ready;\n newClip.tickInterceptor = this.tickInterceptor;\n return newClip;\n }\n\n #removeVideoSamples(startTime: number, endTime: number): ExtMP4Sample[] {\n if (this.#videoSamples.length === 0) return [];\n\n const beforeSamples: ExtMP4Sample[] = [];\n const afterSamples: ExtMP4Sample[] = [];\n\n let lastBeforeIDRIndex = -1;\n let firstAfterIDRIndex = -1;\n\n // 首先找到删除区间前的最后一个 IDR 帧和删除区间后的第一个 IDR 帧\n for (let i = 0; i < this.#videoSamples.length; i++) {\n const s = this.#videoSamples[i];\n if (s.is_idr) {\n if (s.cts < startTime) {\n lastBeforeIDRIndex = i;\n } else if (s.cts >= endTime && firstAfterIDRIndex === -1) {\n firstAfterIDRIndex = i;\n break;\n }\n }\n }\n\n // 如果没有找到合适的 IDR 帧，使用最近的 IDR 帧\n if (lastBeforeIDRIndex === -1) {\n // 找到第一个 IDR 帧\n for (let i = 0; i < this.#videoSamples.length; i++) {\n if (this.#videoSamples[i].is_idr) {\n lastBeforeIDRIndex = i;\n break;\n }\n }\n }\n\n if (firstAfterIDRIndex === -1) {\n // 使用最后一个 IDR 帧\n for (let i = this.#videoSamples.length - 1; i >= 0; i--) {\n if (this.#videoSamples[i].is_idr) {\n firstAfterIDRIndex = i;\n break;\n }\n }\n }\n\n // 收集删除区间前的样本（从最后一个 IDR 帧开始）\n if (lastBeforeIDRIndex !== -1) {\n for (let i = lastBeforeIDRIndex; i < this.#videoSamples.length; i++) {\n const s = this.#videoSamples[i];\n if (s.cts >= startTime) break;\n beforeSamples.push({ ...s });\n }\n }\n\n // 收集删除区间后的样本（从第一个 IDR 帧开始）\n if (firstAfterIDRIndex !== -1) {\n const timeOffset = endTime - startTime;\n for (let i = firstAfterIDRIndex; i < this.#videoSamples.length; i++) {\n const s = this.#videoSamples[i];\n afterSamples.push({\n ...s,\n cts: s.cts - timeOffset,\n });\n }\n }\n\n // 合并前后样本\n const mergedSamples = [...beforeSamples, ...afterSamples];\n\n // 确保第一个样本是 IDR 帧\n if (mergedSamples.length > 0 && !mergedSamples[0].is_idr) {\n this.#log.warn(\n 'First sample is not IDR frame after merging, samples might be corrupted',\n );\n }\n\n return mergedSamples;\n }\n\n // 在 MP4Clip 类中添加\n // 在 MP4Clip 类中添加\n setVolume(newVolume: number): void {\n if (newVolume < 0 || newVolume > 1) {\n throw new Error('Volume must be between 0 and 1');\n }\n\n this.#volume = newVolume;\n\n // 通过 tickInterceptor 动态调整音量，避免重新创建解码器\n const originalInterceptor = this.tickInterceptor;\n this.tickInterceptor = async (time, tickRet) => {\n if (tickRet.audio && this.#volume !== 1) {\n for (const channel of tickRet.audio) {\n for (let i = 0; i < channel.length; i++) {\n channel[i] *= this.#volume;\n }\n }\n }\n return originalInterceptor(time, tickRet);\n };\n }\n\n getVolume(): number {\n return this.#volume;\n }\n\n #removeAudioSamples(startTime: number, endTime: number): ExtMP4Sample[] {\n if (this.#audioSamples.length === 0) return [];\n\n const beforeSamples: ExtMP4Sample[] = [];\n const afterSamples: ExtMP4Sample[] = [];\n\n // 分离删除前后的样本\n for (const s of this.#audioSamples) {\n if (s.cts < startTime) {\n beforeSamples.push({ ...s });\n } else if (s.cts >= endTime) {\n // 调整删除后样本的时间戳\n afterSamples.push({\n ...s,\n cts: s.cts - (endTime - startTime),\n });\n }\n // startTime <= s.cts < endTime 的样本被删除\n }\n\n // 合并前后样本\n return [...beforeSamples, ...afterSamples];\n }\n async clone() {\n await this.ready;\n const clip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [...this.#videoSamples],\n audioSamples: [...this.#audioSamples],\n decoderConf: this.#decoderConf,\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await clip.ready;\n clip.tickInterceptor = this.tickInterceptor;\n return clip as this;\n }\n\n /**\n * 拆分 MP4Clip 为仅包含视频轨道和音频轨道的 MP4Clip\n * @returns Mp4CLip[]\n */\n async splitTrack() {\n await this.ready;\n const clips: MP4Clip[] = [];\n if (this.#videoSamples.length > 0) {\n const videoClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [...this.#videoSamples],\n audioSamples: [],\n decoderConf: {\n video: this.#decoderConf.video,\n audio: null,\n },\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await videoClip.ready;\n videoClip.tickInterceptor = this.tickInterceptor;\n clips.push(videoClip);\n }\n if (this.#audioSamples.length > 0) {\n const audioClip = new MP4Clip(\n {\n localFile: this.#localFile,\n videoSamples: [],\n audioSamples: [...this.#audioSamples],\n decoderConf: {\n audio: this.#decoderConf.audio,\n video: null,\n },\n headerBoxPos: this.#headerBoxPos,\n },\n this.#opts,\n );\n await audioClip.ready;\n audioClip.tickInterceptor = this.tickInterceptor;\n clips.push(audioClip);\n }\n\n return clips;\n }\n\n destroy(): void {\n if (this.#destroyed) return;\n this.#log.info('MP4Clip destroy');\n this.#destroyed = true;\n\n this.#videoFrameFinder?.destroy();\n this.#audioFrameFinder?.destroy();\n }\n}\n\nfunction genMeta(\n decoderConf: MP4DecoderConf,\n videoSamples: ExtMP4Sample[],\n audioSamples: ExtMP4Sample[],\n) {\n const meta = {\n duration: 0,\n width: 0,\n height: 0,\n audioSampleRate: 0,\n audioChanCount: 0,\n };\n if (decoderConf.video != null && videoSamples.length > 0) {\n meta.width = decoderConf.video.codedWidth ?? 0;\n meta.height = decoderConf.video.codedHeight ?? 0;\n }\n if (decoderConf.audio != null && audioSamples.length > 0) {\n meta.audioSampleRate = DEFAULT_AUDIO_CONF.sampleRate;\n meta.audioChanCount = DEFAULT_AUDIO_CONF.channelCount;\n }\n\n let vDuration = 0;\n let aDuration = 0;\n if (videoSamples.length > 0) {\n for (let i = videoSamples.length - 1; i >= 0; i--) {\n const s = videoSamples[i];\n if (s.deleted) continue;\n vDuration = s.cts + s.duration;\n break;\n }\n }\n if (audioSamples.length > 0) {\n const lastSampele = audioSamples.at(-1)!;\n aDuration = lastSampele.cts + lastSampele.duration;\n }\n meta.duration = Math.max(vDuration, aDuration);\n\n return meta;\n}\n\nfunction genDecoder(\n decoderConf: MP4DecoderConf,\n localFileReader: LocalFileReader,\n videoSamples: ExtMP4Sample[],\n audioSamples: ExtMP4Sample[],\n volume: number,\n) {\n return {\n audioFrameFinder:\n volume === 0 || decoderConf.audio == null || audioSamples.length === 0\n ? null\n : new AudioFrameFinder(\n localFileReader,\n audioSamples,\n decoderConf.audio,\n {\n volume,\n targetSampleRate: DEFAULT_AUDIO_CONF.sampleRate,\n },\n ),\n videoFrameFinder:\n decoderConf.video == null || videoSamples.length === 0\n ? null\n : new VideoFrameFinder(\n localFileReader,\n videoSamples,\n decoderConf.video,\n ),\n };\n}\n\nasync function mp4FileToSamples(otFile: OPFSToolFile, opts: MP4ClipOpts = {}) {\n let mp4Info: MP4Info | null = null;\n const decoderConf: MP4DecoderConf = { video: null, audio: null };\n let videoSamples: ExtMP4Sample[] = [];\n let audioSamples: ExtMP4Sample[] = [];\n let headerBoxPos: Array<{ start: number; size: number }> = [];\n\n let videoDeltaTS = -1;\n let audioDeltaTS = -1;\n const reader = await otFile.createReader();\n await quickParseMP4File(\n reader,\n (data) => {\n mp4Info = data.info;\n const ftyp = data.mp4boxFile.ftyp!;\n headerBoxPos.push({ start: ftyp.start, size: ftyp.size });\n const moov = data.mp4boxFile.moov!;\n headerBoxPos.push({ start: moov.start, size: moov.size });\n\n let { videoDecoderConf: vc, audioDecoderConf: ac } = extractFileConfig(\n data.mp4boxFile,\n data.info,\n );\n decoderConf.video = vc ?? null;\n decoderConf.audio = ac ?? null;\n if (vc == null && ac == null) {\n Log.error('MP4Clip no video and audio track');\n }\n Log.info(\n 'mp4BoxFile moov ready',\n {\n ...data.info,\n tracks: null,\n videoTracks: null,\n audioTracks: null,\n },\n decoderConf,\n );\n },\n (_, type, samples) => {\n if (type === 'video') {\n if (videoDeltaTS === -1) videoDeltaTS = samples[0].dts;\n for (const s of samples) {\n videoSamples.push(normalizeTimescale(s, videoDeltaTS, 'video'));\n }\n } else if (type === 'audio' && opts.audio) {\n if (audioDeltaTS === -1) audioDeltaTS = samples[0].dts;\n for (const s of samples) {\n audioSamples.push(normalizeTimescale(s, audioDeltaTS, 'audio'));\n }\n }\n },\n );\n await reader.close();\n\n const lastSampele = videoSamples.at(-1) ?? audioSamples.at(-1);\n if (mp4Info == null) {\n throw Error('MP4Clip stream is done, but not emit ready');\n } else if (lastSampele == null) {\n throw Error('MP4Clip stream not contain any sample');\n }\n // 修复首帧黑帧\n fixFirstBlackFrame(videoSamples);\n Log.info('mp4 stream parsed');\n return {\n videoSamples,\n audioSamples,\n decoderConf,\n headerBoxPos,\n };\n\n function normalizeTimescale(\n s: MP4Sample,\n delta = 0,\n sampleType: 'video' | 'audio',\n ) {\n // todo: perf 丢弃多余字段，小尺寸对象性能更好\n const idrOffset =\n sampleType === 'video' && s.is_sync\n ? idrNALUOffset(s.data, s.description.type)\n : -1;\n let offset = s.offset;\n let size = s.size;\n if (idrOffset >= 0) {\n // 当 IDR 帧前面携带 SEI 数据可能导致解码失败\n // 所以此处通过控制 offset、size 字段跳过 SEI 数据\n offset += idrOffset;\n size -= idrOffset;\n }\n return {\n ...s,\n is_idr: idrOffset >= 0,\n offset,\n size,\n cts: ((s.cts - delta) / s.timescale) * 1e6,\n dts: ((s.dts - delta) / s.timescale) * 1e6,\n duration: (s.duration / s.timescale) * 1e6,\n timescale: 1e6,\n // 音频数据量可控，直接保存在内存中\n data: sampleType === 'video' ? null : s.data,\n };\n }\n}\n\nclass VideoFrameFinder {\n #dec: VideoDecoder | null = null;\n constructor(\n public localFileReader: LocalFileReader,\n public samples: ExtMP4Sample[],\n public conf: VideoDecoderConfig,\n ) {}\n\n #ts = 0;\n #curAborter = { abort: false, st: performance.now() };\n find = async (time: number): Promise<VideoFrame | null> => {\n if (\n this.#dec == null ||\n this.#dec.state === 'closed' ||\n time <= this.#ts ||\n time - this.#ts > 3e6\n ) {\n this.#reset(time);\n }\n\n this.#curAborter.abort = true;\n this.#ts = time;\n\n this.#curAborter = { abort: false, st: performance.now() };\n const vf = await this.#parseFrame(time, this.#dec, this.#curAborter);\n this.#sleepCnt = 0;\n return vf;\n };\n\n // fix VideoFrame duration is null\n #lastVfDur = 0;\n\n #downgradeSoftDecode = false;\n #videoDecCusorIdx = 0;\n #videoFrames: VideoFrame[] = [];\n #outputFrameCnt = 0;\n #inputChunkCnt = 0;\n #sleepCnt = 0;\n #predecodeErr = false;\n #parseFrame = async (\n time: number,\n dec: VideoDecoder | null,\n aborter: { abort: boolean; st: number },\n ): Promise<VideoFrame | null> => {\n if (dec == null || dec.state === 'closed' || aborter.abort) return null;\n\n if (this.#videoFrames.length > 0) {\n const vf = this.#videoFrames[0];\n if (time < vf.timestamp) return null;\n // 弹出第一帧\n this.#videoFrames.shift();\n // 第一帧过期，找下一帧\n if (time > vf.timestamp + (vf.duration ?? 0)) {\n vf.close();\n return await this.#parseFrame(time, dec, aborter);\n }\n\n if (!this.#predecodeErr && this.#videoFrames.length < 10) {\n // 预解码避免等待\n this.#startDecode(dec).catch((err) => {\n this.#predecodeErr = true;\n this.#reset(time);\n throw err;\n });\n }\n // 符合期望\n return vf;\n }\n\n // 缺少帧数据\n if (\n this.#decoding ||\n (this.#outputFrameCnt < this.#inputChunkCnt && dec.decodeQueueSize > 0)\n ) {\n if (performance.now() - aborter.st > 6e3) {\n throw Error(\n `MP4Clip.tick video timeout, ${JSON.stringify(this.#getState())}`,\n );\n }\n // 解码中，等待，然后重试\n this.#sleepCnt += 1;\n await sleep(15);\n } else if (this.#videoDecCusorIdx >= this.samples.length) {\n // decode completed\n return null;\n } else {\n try {\n await this.#startDecode(dec);\n } catch (err) {\n this.#reset(time);\n throw err;\n }\n }\n return await this.#parseFrame(time, dec, aborter);\n };\n\n #decoding = false;\n #startDecode = async (dec: VideoDecoder) => {\n if (this.#decoding || dec.decodeQueueSize > 600) return;\n\n // 启动解码任务，然后重试\n let endIdx = this.#videoDecCusorIdx + 1;\n if (endIdx > this.samples.length) return;\n\n this.#decoding = true;\n // 该 GoP 时间区间有时间匹配，且未被删除的帧\n let hasValidFrame = false;\n for (; endIdx < this.samples.length; endIdx++) {\n const s = this.samples[endIdx];\n if (!hasValidFrame && !s.deleted) {\n hasValidFrame = true;\n }\n // 找一个 GoP，所以是下一个 IDR 帧结束\n if (s.is_idr) break;\n }\n\n if (hasValidFrame) {\n const samples = this.samples.slice(this.#videoDecCusorIdx, endIdx);\n if (samples[0]?.is_idr !== true) {\n Log.warn('First sample not idr frame');\n } else {\n const readStarTime = performance.now();\n const chunks = await videosamples2Chunks(samples, this.localFileReader);\n\n const readCost = performance.now() - readStarTime;\n if (readCost > 1000) {\n const first = samples[0];\n const last = samples.at(-1)!;\n const rangSize = last.offset + last.size - first.offset;\n Log.warn(\n `Read video samples time cost: ${Math.round(readCost)}ms, file chunk size: ${rangSize}`,\n );\n }\n // Wait for the previous asynchronous operation to complete, at which point the task may have already been terminated\n if (dec.state === 'closed') return;\n\n this.#lastVfDur = chunks[0]?.duration ?? 0;\n decodeGoP(dec, chunks, {\n onDecodingError: (err) => {\n if (this.#downgradeSoftDecode) {\n throw err;\n } else if (this.#outputFrameCnt === 0) {\n this.#downgradeSoftDecode = true;\n Log.warn('Downgrade to software decode');\n this.#reset();\n }\n },\n });\n\n this.#inputChunkCnt += chunks.length;\n }\n }\n this.#videoDecCusorIdx = endIdx;\n this.#decoding = false;\n };\n\n #reset = (time?: number) => {\n this.#decoding = false;\n this.#videoFrames.forEach((f) => f.close());\n this.#videoFrames = [];\n if (time == null || time === 0) {\n this.#videoDecCusorIdx = 0;\n } else {\n let keyIdx = 0;\n for (let i = 0; i < this.samples.length; i++) {\n const s = this.samples[i];\n if (s.is_idr) keyIdx = i;\n if (s.cts < time) continue;\n this.#videoDecCusorIdx = keyIdx;\n break;\n }\n }\n this.#inputChunkCnt = 0;\n this.#outputFrameCnt = 0;\n if (this.#dec?.state !== 'closed') this.#dec?.close();\n const encoderConf = {\n ...this.conf,\n ...(this.#downgradeSoftDecode\n ? { hardwareAcceleration: 'prefer-software' }\n : {}),\n } as VideoDecoderConfig;\n this.#dec = new VideoDecoder({\n output: (vf) => {\n this.#outputFrameCnt += 1;\n if (vf.timestamp === -1) {\n vf.close();\n return;\n }\n let rsVf = vf;\n if (vf.duration == null) {\n rsVf = new VideoFrame(vf, {\n duration: this.#lastVfDur,\n });\n vf.close();\n }\n this.#videoFrames.push(rsVf);\n },\n error: (err) => {\n if (err.message.includes('Codec reclaimed due to inactivity')) {\n // todo: 因无活动被自动关闭的解码器，是否需要自动重启？\n this.#dec = null;\n Log.warn(err.message);\n return;\n }\n\n const errMsg = `VideoFinder VideoDecoder err: ${err.message}, config: ${JSON.stringify(encoderConf)}, state: ${JSON.stringify(this.#getState())}`;\n Log.error(errMsg);\n throw Error(errMsg);\n },\n });\n this.#dec.configure(encoderConf);\n };\n\n #getState = () => ({\n time: this.#ts,\n decState: this.#dec?.state,\n decQSize: this.#dec?.decodeQueueSize,\n decCusorIdx: this.#videoDecCusorIdx,\n sampleLen: this.samples.length,\n inputCnt: this.#inputChunkCnt,\n outputCnt: this.#outputFrameCnt,\n cacheFrameLen: this.#videoFrames.length,\n softDeocde: this.#downgradeSoftDecode,\n clipIdCnt: CLIP_ID,\n sleepCnt: this.#sleepCnt,\n memInfo: memoryUsageInfo(),\n });\n\n destroy = () => {\n if (this.#dec?.state !== 'closed') this.#dec?.close();\n this.#dec = null;\n this.#curAborter.abort = true;\n this.#videoFrames.forEach((f) => f.close());\n this.#videoFrames = [];\n this.localFileReader.close();\n };\n}\n\nfunction findIndexOfSamples(time: number, samples: ExtMP4Sample[]) {\n for (let i = 0; i < samples.length; i++) {\n const s = samples[i];\n if (time >= s.cts && time < s.cts + s.duration) {\n return i;\n }\n if (s.cts > time) break;\n }\n return 0;\n}\n\nclass AudioFrameFinder {\n #volume = 1;\n #sampleRate;\n constructor(\n public localFileReader: LocalFileReader,\n public samples: ExtMP4Sample[],\n public conf: AudioDecoderConfig,\n opts: { volume: number; targetSampleRate: number },\n ) {\n this.#volume = opts.volume;\n this.#sampleRate = opts.targetSampleRate;\n }\n\n #dec: ReturnType<typeof createAudioChunksDecoder> | null = null;\n #curAborter = { abort: false, st: performance.now() };\n find = async (time: number): Promise<Float32Array[]> => {\n const needResetTime = time <= this.#ts || time - this.#ts > 0.1e6;\n if (this.#dec == null || this.#dec.state === 'closed' || needResetTime) {\n this.#reset();\n }\n\n if (needResetTime) {\n // 前后获取音频数据差异不能超过 100ms(经验值)，否则视为 seek 操作，重置解码器\n // seek 操作，重置时间\n this.#ts = time;\n this.#decCusorIdx = findIndexOfSamples(time, this.samples);\n }\n\n this.#curAborter.abort = true;\n const deltaTime = time - this.#ts;\n this.#ts = time;\n\n this.#curAborter = { abort: false, st: performance.now() };\n\n const pcmData = await this.#parseFrame(\n Math.ceil(deltaTime * (this.#sampleRate / 1e6)),\n this.#dec,\n this.#curAborter,\n );\n this.#sleepCnt = 0;\n return pcmData;\n };\n\n #ts = 0;\n #decCusorIdx = 0;\n #pcmData: {\n frameCnt: number;\n data: [Float32Array, Float32Array][];\n } = {\n frameCnt: 0,\n data: [],\n };\n #sleepCnt = 0;\n #parseFrame = async (\n emitFrameCnt: number,\n dec: ReturnType<typeof createAudioChunksDecoder> | null = null,\n aborter: { abort: boolean; st: number },\n ): Promise<Float32Array[]> => {\n if (\n dec == null ||\n aborter.abort ||\n dec.state === 'closed' ||\n emitFrameCnt === 0\n ) {\n return [];\n }\n\n // 数据满足需要\n const ramainFrameCnt = this.#pcmData.frameCnt - emitFrameCnt;\n if (ramainFrameCnt > 0) {\n // 剩余音频数据小于 100ms，预先解码\n if (ramainFrameCnt < DEFAULT_AUDIO_CONF.sampleRate / 10) {\n th