tav-media/lib/src/wasm/movie-audio-reader.js

Cross platform media editing framework

var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
/* eslint-disable no-param-reassign */
import { Http } from '../io/http';
import { tav } from '../tav';
import { movieAudioProcessorCode } from './audio-worklet/movie-audio-processor';
import { setMediaSource } from './media-element-utils';
const DEFAULT_SAMPLE_RATE = 44100;
export class MovieAudioReader {
    constructor(path) {
        this.contentTime = 0;
        this.released = false;
        this.inputChannelCount = 2;
        this.targetBitDepth = 16;
        this.inputSampleRate = DEFAULT_SAMPLE_RATE;
        this.audioSampleCount = 0;
        this.silenceAudioSampleCount = 1024;
        this.maxOutSampleCount = 1024;
        this.adjustRange = 0.1;
        this.audioData = [];
        this.playbackRate = 1;
        this.paused = false;
        this.audioPaused = true;
        this.audioIndex = -1;
        this.type = 1;
        this.audioContextResume = false;
        this.audioInitializationSuccess = false;
        this.lastPostAudioTime = 0;
        this.path = path;
        this.targetBitDepth = 16;
        this.silenceAudioSampleCount = 1024;
        this.maxOutSampleCount = 1024;
        this.audioData = [];
        this.audioSampleCount = 0;
        this.lastPostAudioTime = 0;
        MovieAudioReader.audioReaders.push(this);
        this.audioPromise = new Promise((resolve) => {
            MovieAudioReader.preload(this.path, true).then((preloadContext) => {
                this.preloadContext = preloadContext;
                this.audioContext = preloadContext.audioContext;
                this.resume(resolve);
            });
        });
    }
    static preload(path, useNow = false) {
        return __awaiter(this, void 0, void 0, function* () {
            if (!MovieAudioReader.audioContext) {
                if (!MovieAudioReader.audioProcessorLoaded) {
                    const blob = new Blob([movieAudioProcessorCode], { type: 'text/javascript' });
                    this.processorCodePath = URL.createObjectURL(blob);
                    MovieAudioReader.audioProcessorLoaded = true;
                }
                const audioContext = new AudioContext({ sampleRate: DEFAULT_SAMPLE_RATE });
                yield audioContext.audioWorklet.addModule(this.processorCodePath);
                const audioWorkletNode = new AudioWorkletNode(audioContext, 'movie-audio-processor', { numberOfInputs: MovieAudioReader.maxAudioInputs });
                MovieAudioReader.availableAudioIndexes = Array.from(new Array(MovieAudioReader.maxAudioInputs).keys());
                audioWorkletNode.port.onmessage = (e) => {
                    MovieAudioReader.audioReaders.forEach((reader) => {
                        const audioData = e.data.outDataList[reader.audioIndex];
                        if (!audioData)
                            return;
                        if (reader.audioPaused || reader.audioEl.readyState < 1 || reader.audioEl.ended) {
                            // console.log('audio send data when paused: ', reader.audioIndex, reader.path);
                            return;
                        }
                        const dataArray = Array.from(new Int16Array(audioData.bytes));
                        if (reader.lastPostAudioTime === reader.audioEl.currentTime
                            && Math.max(...dataArray) === 0 && Math.min(...dataArray) === 0) {
                            // console.log(`audio send silence data, index: ${reader.audioIndex}, time: ${reader.audioEl.currentTime}`);
                            return;
                        }
                        reader.lastPostAudioTime = reader.audioEl.currentTime;
                        reader.audioData.push(...dataArray);
                        reader.audioSampleCount += audioData.sampleCount;
                        reader.inputChannelCount = audioData.channelCount;
                    });
                };
                const context = {
                    audioContext,
                    audioWorkletNode,
                    audioEl: null,
                };
                MovieAudioReader.audioContext = context;
            }
            let audioElement = MovieAudioReader.audioElements[path];
            MovieAudioReader.audioElements[path] = null;
            if (!audioElement) {
                audioElement = MovieAudioReader.createAudioElement(path);
            }
            if (!useNow) {
                yield MovieAudioReader.audioContext.audioContext.suspend();
                MovieAudioReader.audioElements[path] = audioElement;
            }
            return Object.assign(Object.assign({}, MovieAudioReader.audioContext), { audioEl: audioElement });
        });
    }
    static createAudioElement(path) {
        const audioEl = document.createElement('audio');
        audioEl.preload = 'auto';
        audioEl.crossOrigin = 'anonymous';
        const src = Http.getUrl(path);
        setMediaSource(audioEl, src);
        const source = MovieAudioReader.audioContext.audioContext.createMediaElementSource(audioEl);
        const audioElement = {
            audioEl,
            source,
            initializationSuccess: true,
        };
        audioEl.onerror = () => {
            audioElement.initializationSuccess = false;
        };
        return audioElement;
    }
    static pauseAll() {
        MovieAudioReader.pauseTimeout = setTimeout(() => {
            var _a;
            (_a = MovieAudioReader.audioContext) === null || _a === void 0 ? void 0 : _a.audioWorkletNode.port.postMessage({ status: 'pause' });
        }, 10 * 1000);
    }
    static playAll() {
        var _a;
        if (MovieAudioReader.pauseTimeout)
            clearTimeout(MovieAudioReader.pauseTimeout);
        (_a = MovieAudioReader.audioContext) === null || _a === void 0 ? void 0 : _a.audioWorkletNode.port.postMessage({ status: 'play' });
    }
    static MakeFromPath(path) {
        return new MovieAudioReader(path);
    }
    static MakeFromBytes(bytesOffset, length) {
        const module = tav;
        const arrayBuffer = new Uint8Array(module.HEAPU8.buffer, bytesOffset, length);
        const newArrayBuffer = arrayBuffer.slice(0, length);
        const blob = new Blob([newArrayBuffer.buffer]);
        const path = URL.createObjectURL(blob);
        return new MovieAudioReader(path);
    }
    render() {
        if (this.released)
            return;
        requestAnimationFrame(() => this.render());
        if (!this.audioEl.paused
            && this.audioEl.currentTime - this.contentTime > 2 * this.adjustRange * this.playbackRate) {
            if (this.audioInitializationSuccess)
                this.pauseAudio();
            return;
        }
        if (this.audioEl.paused && this.audioEl.currentTime - this.contentTime <= this.adjustRange * this.playbackRate) {
            if (this.audioInitializationSuccess)
                this.playAudio();
        }
    }
    readNextSample() {
        const module = tav;
        let numSamples = Math.min(this.audioSampleCount, this.maxOutSampleCount);
        const byteCount = Math.ceil(numSamples * this.inputChannelCount);
        let outData = [];
        if (this.audioData.length === 0 || this.paused) {
            outData = Array.from(new Int16Array(new Array(this.silenceAudioSampleCount * this.inputChannelCount)));
            numSamples = this.silenceAudioSampleCount;
        }
        else {
            outData = this.audioData.splice(0, byteCount);
            this.audioSampleCount -= numSamples;
        }
        this.contentTime += (numSamples / this.inputSampleRate) * this.playbackRate;
        const audioBuffer = new Int16Array(outData);
        const audioUnit8Buffer = new Uint8Array(audioBuffer.buffer);
        const numBytes = audioUnit8Buffer.byteLength * Uint8Array.BYTES_PER_ELEMENT;
        const dataPtr = module._malloc(numBytes);
        if (audioBuffer.length > 0) {
            this.lastBufferData = dataPtr;
        }
        const dataOnHeap = new Uint8Array(module.HEAPU8.buffer, dataPtr, numBytes);
        dataOnHeap.set(audioUnit8Buffer);
        return {
            bytes: dataOnHeap.byteOffset,
            length: dataOnHeap.length,
            sampleRate: this.inputSampleRate,
            channels: this.inputChannelCount,
            outputSamplesCount: numSamples,
        };
    }
    decodeAudio() {
        return this.audioPromise;
    }
    setOptions(options) {
        if (!this.audioContextResume) {
            this.resume(() => { });
            return;
        }
        if (options.playbackRate) {
            if (this.playbackRate !== options.playbackRate) {
                this.playbackRate = options.playbackRate;
                if (options.playbackRate > 4 || options.playbackRate < 0.1) {
                    console.warn(`Invalid audio playback rate ${options.playbackRate.toFixed(2)}! Please set playback rate between 0.1 and 4.0`);
                }
                const playbackRate = Math.max(Math.min(this.playbackRate, 4), 0.1);
                this.audioEl.playbackRate = playbackRate;
            }
            if (this.audioEl.muted) {
                this.audioEl.muted = false;
            }
        }
        else {
            this.audioEl.muted = true;
        }
    }
    freeBuffer() {
        if (!this.lastBufferData) {
            return;
        }
        const module = tav;
        module._free(this.lastBufferData);
        this.lastBufferData = null;
    }
    pause() {
        return new Promise((resolve) => {
            this.seekTo(this.contentTime * 1000000, true);
            this.paused = true;
            resolve(true);
        });
    }
    continue() {
        this.paused = false;
        return Promise.resolve(true);
    }
    seekTo(contentTime, forceSeek = false) {
        var _a;
        if (!this.audioContextResume) {
            return;
        }
        if ((_a = this.audioEl) === null || _a === void 0 ? void 0 : _a.paused) {
            if (this.audioInitializationSuccess)
                this.playAudio();
        }
        const targetTime = contentTime / 1000000;
        if (Math.abs(targetTime - this.contentTime) < this.adjustRange * this.playbackRate && !forceSeek) {
            return;
        }
        this.audioData = [];
        this.audioSampleCount = 0;
        this.contentTime = targetTime;
        if (this.audioEl)
            this.audioEl.currentTime = targetTime;
    }
    release() {
        MovieAudioReader.audioReaders.splice(MovieAudioReader.audioReaders.indexOf(this), 1);
        this.freeBuffer();
        this.released = true;
        this.audioData = [];
        this.audioSampleCount = 0;
        this.contentTime = 0;
        if (!this.audioEl)
            return;
        this.audioEl.currentTime = 0;
        this.pauseAudio();
        this.audioWorkletNode.port.postMessage({ operation: 'disconnect', index: this.audioIndex });
        this.source.disconnect();
        MovieAudioReader.availableAudioIndexes.push(this.audioIndex);
        this.audioIndex = -1;
        if (!MovieAudioReader.audioElements[this.path]) {
            MovieAudioReader.audioElements[this.path] = {
                audioEl: this.audioEl,
                source: this.source,
                initializationSuccess: true,
            };
        }
        else {
            this.audioEl.remove();
        }
    }
    playAudio() {
        this.audioEl.play()
            .then(() => {
            this.audioPaused = false;
        })
            .catch(() => { });
    }
    pauseAudio() {
        var _a;
        (_a = this.audioEl) === null || _a === void 0 ? void 0 : _a.pause();
        this.audioPaused = true;
    }
    resume(resolve) {
        let isCallback = false;
        this.audioContext.resume().then(() => {
            if (this.audioContextResume) {
                return;
            }
            isCallback = true;
            this.audioContextResume = true;
            this.audioEl = this.preloadContext.audioEl.audioEl;
            this.audioInitializationSuccess = this.preloadContext.audioEl.initializationSuccess;
            this.source = this.preloadContext.audioEl.source;
            this.audioWorkletNode = this.preloadContext.audioWorkletNode;
            if (MovieAudioReader.availableAudioIndexes.length === 0) {
                console.warn(`too many audios at same time, maximum ${MovieAudioReader.maxAudioInputs}!`);
                this.audioIndex = 0;
            }
            else {
                this.audioIndex = MovieAudioReader.availableAudioIndexes.shift();
            }
            this.source.connect(this.audioWorkletNode, 0, this.audioIndex);
            this.audioWorkletNode.port.postMessage({ operation: 'connect', index: this.audioIndex });
            this.inputSampleRate = this.audioContext.sampleRate;
            this.inputChannelCount = this.source.channelCount;
            const bitDepth = this.audioWorkletNode.parameters.get('bitDepth');
            bitDepth.setValueAtTime(this.targetBitDepth, 0);
            this.audioEl.currentTime = 0;
            this.audioEl.playbackRate = this.playbackRate;
            if (this.audioInitializationSuccess)
                this.playAudio();
            requestAnimationFrame(() => this.render());
            resolve(this);
        });
        setTimeout(() => {
            if (!isCallback) {
                resolve(this);
            }
        }, 16);
    }
}
MovieAudioReader.audioProcessorLoaded = false;
MovieAudioReader.maxAudioInputs = 100;
MovieAudioReader.audioElements = {};
MovieAudioReader.audioReaders = [];