remotion/dist/cjs/audio/streaming-pitch-shifter.js

Make videos programmatically

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.StreamingPitchShifter = void 0;
// The pitch-shifting algorithm is adapted from Vanilagy's WSOLA audio
// stretcher: https://gist.github.com/Vanilagy/05f7901f4c4398356657e3a86c7aee05
const REFERENCE_SAMPLE_RATE = 48000;
const REFERENCE_HOP_SIZE = 512;
const makePlanarAudio = (numberOfChannels, length) => {
    return new Array(numberOfChannels)
        .fill(null)
        .map(() => new Float32Array(length));
};
const ensurePlanarCapacity = ({ buffers, requiredLength, }) => {
    if (buffers[0].length >= requiredLength) {
        return buffers;
    }
    let newLength = buffers[0].length;
    while (newLength < requiredLength) {
        newLength *= 2;
    }
    return buffers.map((buffer) => {
        const expanded = new Float32Array(newLength);
        expanded.set(buffer);
        return expanded;
    });
};
class PlanarAudioQueue {
    chunks = [];
    length = 0;
    push(audio) {
        if (audio[0].length === 0) {
            return;
        }
        this.chunks.push(audio);
        this.length += audio[0].length;
    }
    take(numberOfFrames, numberOfChannels) {
        const framesToTake = Math.min(numberOfFrames, this.length);
        const result = makePlanarAudio(numberOfChannels, framesToTake);
        let written = 0;
        while (written < framesToTake) {
            const first = this.chunks[0];
            const available = first[0].length;
            const count = Math.min(available, framesToTake - written);
            for (let channel = 0; channel < numberOfChannels; channel++) {
                result[channel].set(first[channel].subarray(0, count), written);
            }
            if (count === available) {
                this.chunks.shift();
            }
            else {
                this.chunks[0] = first.map((channel) => channel.subarray(count));
            }
            written += count;
            this.length -= count;
        }
        return result;
    }
    getLength() {
        return this.length;
    }
}
// A streaming implementation of Waveform Similarity Overlap-Add. It changes
// duration while retaining pitch. Pitch shifting is achieved by following this
// stage with a resampler that restores the original duration.
class StreamingTimeStretcher {
    numberOfChannels;
    factor;
    hopSize;
    windowSize;
    searchRadius;
    analysisHop;
    input;
    inputLength = 0;
    output;
    outputLength = 0;
    analysisPosition = 0;
    synthesisPosition = 0;
    initialized = false;
    finalized = false;
    totalInputFrames = 0;
    totalOutputFrames = 0;
    constructor({ numberOfChannels, sampleRate, factor, }) {
        this.numberOfChannels = numberOfChannels;
        this.factor = factor;
        this.hopSize = Math.max(32, Math.round((REFERENCE_HOP_SIZE * sampleRate) / REFERENCE_SAMPLE_RATE));
        this.windowSize = this.hopSize * 2;
        this.searchRadius = this.hopSize;
        this.analysisHop = this.hopSize / factor;
        this.input = makePlanarAudio(numberOfChannels, 65536);
        this.output = makePlanarAudio(numberOfChannels, 65536);
    }
    append(audio) {
        if (this.finalized) {
            throw new Error('Cannot append audio after the time stretcher was finalized.');
        }
        const { length } = audio[0];
        this.input = ensurePlanarCapacity({
            buffers: this.input,
            requiredLength: this.inputLength + length,
        });
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            this.input[channel].set(audio[channel], this.inputLength);
        }
        this.inputLength += length;
        this.totalInputFrames += length;
        this.process();
        return this.drainFinalizedOutput();
    }
    findBestAnalysisPosition({ expectedPosition, nextSynthesisPosition, }) {
        const minimum = Math.max(0, Math.floor(expectedPosition - this.searchRadius));
        const maximum = Math.min(this.inputLength - this.windowSize, Math.ceil(expectedPosition + this.searchRadius));
        let bestPosition = minimum;
        let bestCorrelation = -Infinity;
        for (let candidate = minimum; candidate <= maximum; candidate += 4) {
            let dotProduct = 0;
            let previousEnergy = 0;
            let candidateEnergy = 0;
            for (let channel = 0; channel < this.numberOfChannels; channel++) {
                const previous = this.output[channel];
                const incoming = this.input[channel];
                for (let frame = 0; frame < this.hopSize; frame += 2) {
                    const previousValue = previous[nextSynthesisPosition + frame];
                    const candidateValue = incoming[candidate + frame];
                    dotProduct += previousValue * candidateValue;
                    previousEnergy += previousValue * previousValue;
                    candidateEnergy += candidateValue * candidateValue;
                }
            }
            const correlation = dotProduct /
                (Math.sqrt(previousEnergy * candidateEnergy) || Number.EPSILON);
            if (correlation > bestCorrelation) {
                bestCorrelation = correlation;
                bestPosition = candidate;
            }
        }
        const fineMinimum = Math.max(minimum, bestPosition - 4);
        const fineMaximum = Math.min(maximum, bestPosition + 4);
        for (let candidate = fineMinimum; candidate <= fineMaximum; candidate++) {
            let dotProduct = 0;
            let previousEnergy = 0;
            let candidateEnergy = 0;
            for (let channel = 0; channel < this.numberOfChannels; channel++) {
                const previous = this.output[channel];
                const incoming = this.input[channel];
                for (let frame = 0; frame < this.hopSize; frame++) {
                    const previousValue = previous[nextSynthesisPosition + frame];
                    const candidateValue = incoming[candidate + frame];
                    dotProduct += previousValue * candidateValue;
                    previousEnergy += previousValue * previousValue;
                    candidateEnergy += candidateValue * candidateValue;
                }
            }
            const correlation = dotProduct /
                (Math.sqrt(previousEnergy * candidateEnergy) || Number.EPSILON);
            if (correlation > bestCorrelation) {
                bestCorrelation = correlation;
                bestPosition = candidate;
            }
        }
        return bestPosition;
    }
    process() {
        if (!this.initialized) {
            if (this.inputLength < this.windowSize + this.searchRadius) {
                return;
            }
            for (let channel = 0; channel < this.numberOfChannels; channel++) {
                this.output[channel].set(this.input[channel].subarray(0, this.windowSize));
            }
            this.outputLength = this.windowSize;
            this.initialized = true;
        }
        while (true) {
            const expectedPosition = this.analysisPosition + this.analysisHop;
            if (expectedPosition + this.searchRadius + this.windowSize >
                this.inputLength) {
                break;
            }
            const nextSynthesisPosition = this.synthesisPosition + this.hopSize;
            this.output = ensurePlanarCapacity({
                buffers: this.output,
                requiredLength: nextSynthesisPosition + this.windowSize,
            });
            const bestPosition = this.findBestAnalysisPosition({
                expectedPosition,
                nextSynthesisPosition,
            });
            for (let channel = 0; channel < this.numberOfChannels; channel++) {
                for (let frame = 0; frame < this.hopSize; frame++) {
                    const fadeIn = 0.5 - 0.5 * Math.cos((Math.PI * (frame + 1)) / (this.hopSize + 1));
                    const outputIndex = nextSynthesisPosition + frame;
                    this.output[channel][outputIndex] =
                        this.output[channel][outputIndex] * (1 - fadeIn) +
                            this.input[channel][bestPosition + frame] * fadeIn;
                }
                this.output[channel].set(this.input[channel].subarray(bestPosition + this.hopSize, bestPosition + this.windowSize), nextSynthesisPosition + this.hopSize);
            }
            // Keep the analysis clock independent from the correlation correction.
            // Periodic signals can have equally good matches at an earlier period; if
            // the correction became the next clock position, the iterator could stop
            // making forward progress.
            this.analysisPosition = expectedPosition;
            this.synthesisPosition = nextSynthesisPosition;
            this.outputLength = nextSynthesisPosition + this.windowSize;
        }
    }
    drainFinalizedOutput() {
        if (!this.initialized) {
            return makePlanarAudio(this.numberOfChannels, 0);
        }
        const finalizedLength = Math.max(0, this.synthesisPosition + this.hopSize);
        const result = this.output.map((channel) => channel.slice(0, finalizedLength));
        this.totalOutputFrames += finalizedLength;
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            this.output[channel].copyWithin(0, finalizedLength, this.outputLength);
        }
        this.outputLength -= finalizedLength;
        this.synthesisPosition -= finalizedLength;
        const inputFramesToDiscard = Math.max(0, Math.floor(this.analysisPosition) - this.searchRadius);
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            this.input[channel].copyWithin(0, inputFramesToDiscard, this.inputLength);
        }
        this.inputLength -= inputFramesToDiscard;
        this.analysisPosition -= inputFramesToDiscard;
        return result;
    }
    finalize() {
        if (this.finalized) {
            throw new Error('The time stretcher has already been finalized.');
        }
        this.finalized = true;
        const targetLength = Math.round(this.totalInputFrames * this.factor);
        const padding = makePlanarAudio(this.numberOfChannels, this.windowSize + this.searchRadius * 2);
        this.input = ensurePlanarCapacity({
            buffers: this.input,
            requiredLength: this.inputLength + padding[0].length,
        });
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            this.input[channel].set(padding[channel], this.inputLength);
        }
        this.inputLength += padding[0].length;
        this.process();
        const finalized = this.drainFinalizedOutput();
        const remaining = Math.max(0, targetLength - this.totalOutputFrames + finalized[0].length);
        if (finalized[0].length >= remaining) {
            return finalized.map((channel) => channel.slice(0, remaining));
        }
        const result = makePlanarAudio(this.numberOfChannels, remaining);
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            result[channel].set(finalized[channel]);
        }
        return result;
    }
}
class StreamingLinearResampler {
    numberOfChannels;
    step;
    input;
    inputLength = 0;
    position = 0;
    constructor({ numberOfChannels, step, }) {
        this.numberOfChannels = numberOfChannels;
        this.step = step;
        this.input = makePlanarAudio(numberOfChannels, 65536);
    }
    append(audio) {
        this.input = ensurePlanarCapacity({
            buffers: this.input,
            requiredLength: this.inputLength + audio[0].length,
        });
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            this.input[channel].set(audio[channel], this.inputLength);
        }
        this.inputLength += audio[0].length;
        return this.process(false);
    }
    process(finalizing) {
        const outputLength = Math.max(0, Math.floor((this.inputLength - (finalizing ? 0 : 1) - this.position) / this.step) + 1);
        const result = makePlanarAudio(this.numberOfChannels, outputLength);
        for (let outputFrame = 0; outputFrame < outputLength; outputFrame++) {
            const leftIndex = Math.floor(this.position);
            const rightIndex = Math.min(leftIndex + 1, this.inputLength - 1);
            const fraction = this.position - leftIndex;
            for (let channel = 0; channel < this.numberOfChannels; channel++) {
                const left = this.input[channel][leftIndex];
                const right = this.input[channel][rightIndex];
                result[channel][outputFrame] = left + (right - left) * fraction;
            }
            this.position += this.step;
        }
        const discard = Math.min(Math.floor(this.position), this.inputLength);
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            this.input[channel].copyWithin(0, discard, this.inputLength);
        }
        this.inputLength -= discard;
        this.position -= discard;
        return result;
    }
    finalize() {
        return this.process(true);
    }
}
class StreamingPitchShifter {
    numberOfChannels;
    stretcher;
    resampler;
    outputQueue = new PlanarAudioQueue();
    totalInputFrames = 0;
    totalOutputFrames = 0;
    constructor({ numberOfChannels, sampleRate, toneFrequency, }) {
        this.numberOfChannels = numberOfChannels;
        this.stretcher = new StreamingTimeStretcher({
            numberOfChannels,
            sampleRate,
            factor: toneFrequency,
        });
        this.resampler = new StreamingLinearResampler({
            numberOfChannels,
            step: toneFrequency,
        });
    }
    append(audio) {
        this.totalInputFrames += audio[0].length;
        const stretched = this.stretcher.append(audio);
        this.outputQueue.push(this.resampler.append(stretched));
        return this.takeAvailableOutput();
    }
    takeAvailableOutput() {
        const availableInputFrames = this.totalInputFrames - this.totalOutputFrames;
        const framesToTake = Math.min(availableInputFrames, this.outputQueue.getLength());
        const result = this.outputQueue.take(framesToTake, this.numberOfChannels);
        this.totalOutputFrames += framesToTake;
        return result;
    }
    finalize() {
        this.outputQueue.push(this.resampler.append(this.stretcher.finalize()));
        this.outputQueue.push(this.resampler.finalize());
        const remaining = this.totalInputFrames - this.totalOutputFrames;
        const available = this.outputQueue.take(Math.min(remaining, this.outputQueue.getLength()), this.numberOfChannels);
        const result = makePlanarAudio(this.numberOfChannels, remaining);
        for (let channel = 0; channel < this.numberOfChannels; channel++) {
            result[channel].set(available[channel]);
        }
        this.totalOutputFrames += remaining;
        return result;
    }
}
exports.StreamingPitchShifter = StreamingPitchShifter;