matrix-react-sdk/lib/audio/RecorderWorklet.js

SDK for matrix.org using React

"use strict";

var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = void 0;
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
var _consts = require("./consts");
var _numbers = require("../utils/numbers");
/*
Copyright 2024 New Vector Ltd.
Copyright 2021 The Matrix.org Foundation C.I.C.

SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
Please see LICENSE files in the repository root for full details.
*/

// from AudioWorkletGlobalScope: https://developer.mozilla.org/en-US/docs/Web/API/AudioWorkletGlobalScope

// declare const currentFrame: number;
// declare const sampleRate: number;

// We rate limit here to avoid overloading downstream consumers with amplitude information.
// The two major consumers are the voice message waveform thumbnail (resampled down to an
// appropriate length) and the live waveform shown to the user. Effectively, this controls
// the refresh rate of that live waveform and the number of samples the thumbnail has to
// work with.
const TARGET_AMPLITUDE_FREQUENCY = 16; // Hz

function roundTimeToTargetFreq(seconds) {
  // Epsilon helps avoid floating point rounding issues (1 + 1 = 1.999999, etc)
  return Math.round((seconds + Number.EPSILON) * TARGET_AMPLITUDE_FREQUENCY) / TARGET_AMPLITUDE_FREQUENCY;
}
function nextTimeForTargetFreq(roundedSeconds) {
  // The extra round is just to make sure we cut off any floating point issues
  return roundTimeToTargetFreq(roundedSeconds + 1 / TARGET_AMPLITUDE_FREQUENCY);
}
class MxVoiceWorklet extends AudioWorkletProcessor {
  constructor(...args) {
    super(...args);
    (0, _defineProperty2.default)(this, "nextAmplitudeSecond", 0);
    (0, _defineProperty2.default)(this, "amplitudeIndex", 0);
  }
  process(inputs, outputs, parameters) {
    const currentSecond = roundTimeToTargetFreq(currentTime);
    // We special case the first ping because there's a fairly good chance that we'll miss the zeroth
    // update. Firefox for instance takes 0.06 seconds (roughly) to call this function for the first
    // time. Edge and Chrome occasionally lag behind too, but for the most part are on time.
    //
    // When this doesn't work properly we end up producing a waveform of nulls and no live preview
    // of the recorded message.
    if (currentSecond === this.nextAmplitudeSecond || this.nextAmplitudeSecond === 0) {
      // We're expecting exactly one mono input source, so just grab the very first frame of
      // samples for the analysis.
      const monoChan = inputs[0][0];

      // The amplitude of the frame's samples is effectively the loudness of the frame. This
      // translates into a bar which can be rendered as part of the whole recording clip's
      // waveform.
      //
      // We translate the amplitude down to 0-1 for sanity's sake.
      const minVal = Math.min(...monoChan);
      const maxVal = Math.max(...monoChan);
      const amplitude = (0, _numbers.percentageOf)(maxVal, -1, 1) - (0, _numbers.percentageOf)(minVal, -1, 1);
      this.port.postMessage({
        ev: _consts.PayloadEvent.AmplitudeMark,
        amplitude: amplitude,
        forIndex: this.amplitudeIndex++
      });
      this.nextAmplitudeSecond = nextTimeForTargetFreq(currentSecond);
    }

    // We mostly use this worklet to fire regular clock updates through to components
    this.port.postMessage({
      ev: _consts.PayloadEvent.Timekeep,
      timeSeconds: currentTime
    });

    // We're supposed to return false when we're "done" with the audio clip, but seeing as
    // we are acting as a passive processor we are never truly "done". The browser will clean
    // us up when it is done with us.
    return true;
  }
}
registerProcessor(_consts.WORKLET_NAME, MxVoiceWorklet);
var _default = exports.default = ""; // to appease module loaders (we never use the export)
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJuYW1lcyI6WyJfY29uc3RzIiwicmVxdWlyZSIsIl9udW1iZXJzIiwiVEFSR0VUX0FNUExJVFVERV9GUkVRVUVOQ1kiLCJyb3VuZFRpbWVUb1RhcmdldEZyZXEiLCJzZWNvbmRzIiwiTWF0aCIsInJvdW5kIiwiTnVtYmVyIiwiRVBTSUxPTiIsIm5leHRUaW1lRm9yVGFyZ2V0RnJlcSIsInJvdW5kZWRTZWNvbmRzIiwiTXhWb2ljZVdvcmtsZXQiLCJBdWRpb1dvcmtsZXRQcm9jZXNzb3IiLCJjb25zdHJ1Y3RvciIsImFyZ3MiLCJfZGVmaW5lUHJvcGVydHkyIiwiZGVmYXVsdCIsInByb2Nlc3MiLCJpbnB1dHMiLCJvdXRwdXRzIiwicGFyYW1ldGVycyIsImN1cnJlbnRTZWNvbmQiLCJjdXJyZW50VGltZSIsIm5leHRBbXBsaXR1ZGVTZWNvbmQiLCJtb25vQ2hhbiIsIm1pblZhbCIsIm1pbiIsIm1heFZhbCIsIm1heCIsImFtcGxpdHVkZSIsInBlcmNlbnRhZ2VPZiIsInBvcnQiLCJwb3N0TWVzc2FnZSIsImV2IiwiUGF5bG9hZEV2ZW50IiwiQW1wbGl0dWRlTWFyayIsImZvckluZGV4IiwiYW1wbGl0dWRlSW5kZXgiLCJUaW1la2VlcCIsInRpbWVTZWNvbmRzIiwicmVnaXN0ZXJQcm9jZXNzb3IiLCJXT1JLTEVUX05BTUUiLCJfZGVmYXVsdCIsImV4cG9ydHMiXSwic291cmNlcyI6WyIuLi8uLi9zcmMvYXVkaW8vUmVjb3JkZXJXb3JrbGV0LnRzIl0sInNvdXJjZXNDb250ZW50IjpbIi8qXG5Db3B5cmlnaHQgMjAyNCBOZXcgVmVjdG9yIEx0ZC5cbkNvcHlyaWdodCAyMDIxIFRoZSBNYXRyaXgub3JnIEZvdW5kYXRpb24gQy5JLkMuXG5cblNQRFgtTGljZW5zZS1JZGVudGlmaWVyOiBBR1BMLTMuMC1vbmx5IE9SIEdQTC0zLjAtb25seVxuUGxlYXNlIHNlZSBMSUNFTlNFIGZpbGVzIGluIHRoZSByZXBvc2l0b3J5IHJvb3QgZm9yIGZ1bGwgZGV0YWlscy5cbiovXG5cbmltcG9ydCB7IElBbXBsaXR1ZGVQYXlsb2FkLCBJVGltaW5nUGF5bG9hZCwgUGF5bG9hZEV2ZW50LCBXT1JLTEVUX05BTUUgfSBmcm9tIFwiLi9jb25zdHNcIjtcbmltcG9ydCB7IHBlcmNlbnRhZ2VPZiB9IGZyb20gXCIuLi91dGlscy9udW1iZXJzXCI7XG5cbi8vIGZyb20gQXVkaW9Xb3JrbGV0R2xvYmFsU2NvcGU6IGh0dHBzOi8vZGV2ZWxvcGVyLm1vemlsbGEub3JnL2VuLVVTL2RvY3MvV2ViL0FQSS9BdWRpb1dvcmtsZXRHbG9iYWxTY29wZVxuZGVjbGFyZSBjb25zdCBjdXJyZW50VGltZTogbnVtYmVyO1xuLy8gZGVjbGFyZSBjb25zdCBjdXJyZW50RnJhbWU6IG51bWJlcjtcbi8vIGRlY2xhcmUgY29uc3Qgc2FtcGxlUmF0ZTogbnVtYmVyO1xuXG4vLyBXZSByYXRlIGxpbWl0IGhlcmUgdG8gYXZvaWQgb3ZlcmxvYWRpbmcgZG93bnN0cmVhbSBjb25zdW1lcnMgd2l0aCBhbXBsaXR1ZGUgaW5mb3JtYXRpb24uXG4vLyBUaGUgdHdvIG1ham9yIGNvbnN1bWVycyBhcmUgdGhlIHZvaWNlIG1lc3NhZ2Ugd2F2ZWZvcm0gdGh1bWJuYWlsIChyZXNhbXBsZWQgZG93biB0byBhblxuLy8gYXBwcm9wcmlhdGUgbGVuZ3RoKSBhbmQgdGhlIGxpdmUgd2F2ZWZvcm0gc2hvd24gdG8gdGhlIHVzZXIuIEVmZmVjdGl2ZWx5LCB0aGlzIGNvbnRyb2xzXG4vLyB0aGUgcmVmcmVzaCByYXRlIG9mIHRoYXQgbGl2ZSB3YXZlZm9ybSBhbmQgdGhlIG51bWJlciBvZiBzYW1wbGVzIHRoZSB0aHVtYm5haWwgaGFzIHRvXG4vLyB3b3JrIHdpdGguXG5jb25zdCBUQVJHRVRfQU1QTElUVURFX0ZSRVFVRU5DWSA9IDE2OyAvLyBIelxuXG5mdW5jdGlvbiByb3VuZFRpbWVUb1RhcmdldEZyZXEoc2Vjb25kczogbnVtYmVyKTogbnVtYmVyIHtcbiAgICAvLyBFcHNpbG9uIGhlbHBzIGF2b2lkIGZsb2F0aW5nIHBvaW50IHJvdW5kaW5nIGlzc3VlcyAoMSArIDEgPSAxLjk5OTk5OSwgZXRjKVxuICAgIHJldHVybiBNYXRoLnJvdW5kKChzZWNvbmRzICsgTnVtYmVyLkVQU0lMT04pICogVEFSR0VUX0FNUExJVFVERV9GUkVRVUVOQ1kpIC8gVEFSR0VUX0FNUExJVFVERV9GUkVRVUVOQ1k7XG59XG5cbmZ1bmN0aW9uIG5leHRUaW1lRm9yVGFyZ2V0RnJlcShyb3VuZGVkU2Vjb25kczogbnVtYmVyKTogbnVtYmVyIHtcbiAgICAvLyBUaGUgZXh0cmEgcm91bmQgaXMganVzdCB0byBtYWtlIHN1cmUgd2UgY3V0IG9mZiBhbnkgZmxvYXRpbmcgcG9pbnQgaXNzdWVzXG4gICAgcmV0dXJuIHJvdW5kVGltZVRvVGFyZ2V0RnJlcShyb3VuZGVkU2Vjb25kcyArIDEgLyBUQVJHRVRfQU1QTElUVURFX0ZSRVFVRU5DWSk7XG59XG5cbmNsYXNzIE14Vm9pY2VXb3JrbGV0IGV4dGVuZHMgQXVkaW9Xb3JrbGV0UHJvY2Vzc29yIHtcbiAgICBwcml2YXRlIG5leHRBbXBsaXR1ZGVTZWNvbmQgPSAwO1xuICAgIHByaXZhdGUgYW1wbGl0dWRlSW5kZXggPSAwO1xuXG4gICAgcHVibGljIHByb2Nlc3MoXG4gICAgICAgIGlucHV0czogRmxvYXQzMkFycmF5W11bXSxcbiAgICAgICAgb3V0cHV0czogRmxvYXQzMkFycmF5W11bXSxcbiAgICAgICAgcGFyYW1ldGVyczogUmVjb3JkPHN0cmluZywgRmxvYXQzMkFycmF5PixcbiAgICApOiBib29sZWFuIHtcbiAgICAgICAgY29uc3QgY3VycmVudFNlY29uZCA9IHJvdW5kVGltZVRvVGFyZ2V0RnJlcShjdXJyZW50VGltZSk7XG4gICAgICAgIC8vIFdlIHNwZWNpYWwgY2FzZSB0aGUgZmlyc3QgcGluZyBiZWNhdXNlIHRoZXJlJ3MgYSBmYWlybHkgZ29vZCBjaGFuY2UgdGhhdCB3ZSdsbCBtaXNzIHRoZSB6ZXJvdGhcbiAgICAgICAgLy8gdXBkYXRlLiBGaXJlZm94IGZvciBpbnN0YW5jZSB0YWtlcyAwLjA2IHNlY29uZHMgKHJvdWdobHkpIHRvIGNhbGwgdGhpcyBmdW5jdGlvbiBmb3IgdGhlIGZpcnN0XG4gICAgICAgIC8vIHRpbWUuIEVkZ2UgYW5kIENocm9tZSBvY2Nhc2lvbmFsbHkgbGFnIGJlaGluZCB0b28sIGJ1dCBmb3IgdGhlIG1vc3QgcGFydCBhcmUgb24gdGltZS5cbiAgICAgICAgLy9cbiAgICAgICAgLy8gV2hlbiB0aGlzIGRvZXNuJ3Qgd29yayBwcm9wZXJseSB3ZSBlbmQgdXAgcHJvZHVjaW5nIGEgd2F2ZWZvcm0gb2YgbnVsbHMgYW5kIG5vIGxpdmUgcHJldmlld1xuICAgICAgICAvLyBvZiB0aGUgcmVjb3JkZWQgbWVzc2FnZS5cbiAgICAgICAgaWYgKGN1cnJlbnRTZWNvbmQgPT09IHRoaXMubmV4dEFtcGxpdHVkZVNlY29uZCB8fCB0aGlzLm5leHRBbXBsaXR1ZGVTZWNvbmQgPT09IDApIHtcbiAgICAgICAgICAgIC8vIFdlJ3JlIGV4cGVjdGluZyBleGFjdGx5IG9uZSBtb25vIGlucHV0IHNvdXJjZSwgc28ganVzdCBncmFiIHRoZSB2ZXJ5IGZpcnN0IGZyYW1lIG9mXG4gICAgICAgICAgICAvLyBzYW1wbGVzIGZvciB0aGUgYW5hbHlzaXMuXG4gICAgICAgICAgICBjb25zdCBtb25vQ2hhbiA9IGlucHV0c1swXVswXTtcblxuICAgICAgICAgICAgLy8gVGhlIGFtcGxpdHVkZSBvZiB0aGUgZnJhbWUncyBzYW1wbGVzIGlzIGVmZmVjdGl2ZWx5IHRoZSBsb3VkbmVzcyBvZiB0aGUgZnJhbWUuIFRoaXNcbiAgICAgICAgICAgIC8vIHRyYW5zbGF0ZXMgaW50byBhIGJhciB3aGljaCBjYW4gYmUgcmVuZGVyZWQgYXMgcGFydCBvZiB0aGUgd2hvbGUgcmVjb3JkaW5nIGNsaXAnc1xuICAgICAgICAgICAgLy8gd2F2ZWZvcm0uXG4gICAgICAgICAgICAvL1xuICAgICAgICAgICAgLy8gV2UgdHJhbnNsYXRlIHRoZSBhbXBsaXR1ZGUgZG93biB0byAwLTEgZm9yIHNhbml0eSdzIHNha2UuXG4gICAgICAgICAgICBjb25zdCBtaW5WYWwgPSBNYXRoLm1pbiguLi5tb25vQ2hhbik7XG4gICAgICAgICAgICBjb25zdCBtYXhWYWwgPSBNYXRoLm1heCguLi5tb25vQ2hhbik7XG4gICAgICAgICAgICBjb25zdCBhbXBsaXR1ZGUgPSBwZXJjZW50YWdlT2YobWF4VmFsLCAtMSwgMSkgLSBwZXJjZW50YWdlT2YobWluVmFsLCAtMSwgMSk7XG5cbiAgICAgICAgICAgIHRoaXMucG9ydC5wb3N0TWVzc2FnZSg8SUFtcGxpdHVkZVBheWxvYWQ+e1xuICAgICAgICAgICAgICAgIGV2OiBQYXlsb2FkRXZlbnQuQW1wbGl0dWRlTWFyayxcbiAgICAgICAgICAgICAgICBhbXBsaXR1ZGU6IGFtcGxpdHVkZSxcbiAgICAgICAgICAgICAgICBmb3JJbmRleDogdGhpcy5hbXBsaXR1ZGVJbmRleCsrLFxuICAgICAgICAgICAgfSk7XG4gICAgICAgICAgICB0aGlzLm5leHRBbXBsaXR1ZGVTZWNvbmQgPSBuZXh0VGltZUZvclRhcmdldEZyZXEoY3VycmVudFNlY29uZCk7XG4gICAgICAgIH1cblxuICAgICAgICAvLyBXZSBtb3N0bHkgdXNlIHRoaXMgd29ya2xldCB0byBmaXJlIHJlZ3VsYXIgY2xvY2sgdXBkYXRlcyB0aHJvdWdoIHRvIGNvbXBvbmVudHNcbiAgICAgICAgdGhpcy5wb3J0LnBvc3RNZXNzYWdlKDxJVGltaW5nUGF5bG9hZD57IGV2OiBQYXlsb2FkRXZlbnQuVGltZWtlZXAsIHRpbWVTZWNvbmRzOiBjdXJyZW50VGltZSB9KTtcblxuICAgICAgICAvLyBXZSdyZSBzdXBwb3NlZCB0byByZXR1cm4gZmFsc2Ugd2hlbiB3ZSdyZSBcImRvbmVcIiB3aXRoIHRoZSBhdWRpbyBjbGlwLCBidXQgc2VlaW5nIGFzXG4gICAgICAgIC8vIHdlIGFyZSBhY3RpbmcgYXMgYSBwYXNzaXZlIHByb2Nlc3NvciB3ZSBhcmUgbmV2ZXIgdHJ1bHkgXCJkb25lXCIuIFRoZSBicm93c2VyIHdpbGwgY2xlYW5cbiAgICAgICAgLy8gdXMgdXAgd2hlbiBpdCBpcyBkb25lIHdpdGggdXMuXG4gICAgICAgIHJldHVybiB0cnVlO1xuICAgIH1cbn1cblxucmVnaXN0ZXJQcm9jZXNzb3IoV09SS0xFVF9OQU1FLCBNeFZvaWNlV29ya2xldCk7XG5cbmV4cG9ydCBkZWZhdWx0IFwiXCI7IC8vIHRvIGFwcGVhc2UgbW9kdWxlIGxvYWRlcnMgKHdlIG5ldmVyIHVzZSB0aGUgZXhwb3J0KVxuIl0sIm1hcHBpbmdzIjoiOzs7Ozs7OztBQVFBLElBQUFBLE9BQUEsR0FBQUMsT0FBQTtBQUNBLElBQUFDLFFBQUEsR0FBQUQsT0FBQTtBQVRBO0FBQ0E7QUFDQTtBQUNBO0FBQ0E7QUFDQTtBQUNBOztBQUtBOztBQUVBO0FBQ0E7O0FBRUE7QUFDQTtBQUNBO0FBQ0E7QUFDQTtBQUNBLE1BQU1FLDBCQUEwQixHQUFHLEVBQUUsQ0FBQyxDQUFDOztBQUV2QyxTQUFTQyxxQkFBcUJBLENBQUNDLE9BQWUsRUFBVTtFQUNwRDtFQUNBLE9BQU9DLElBQUksQ0FBQ0MsS0FBSyxDQUFDLENBQUNGLE9BQU8sR0FBR0csTUFBTSxDQUFDQyxPQUFPLElBQUlOLDBCQUEwQixDQUFDLEdBQUdBLDBCQUEwQjtBQUMzRztBQUVBLFNBQVNPLHFCQUFxQkEsQ0FBQ0MsY0FBc0IsRUFBVTtFQUMzRDtFQUNBLE9BQU9QLHFCQUFxQixDQUFDTyxjQUFjLEdBQUcsQ0FBQyxHQUFHUiwwQkFBMEIsQ0FBQztBQUNqRjtBQUVBLE1BQU1TLGNBQWMsU0FBU0MscUJBQXFCLENBQUM7RUFBQUMsWUFBQSxHQUFBQyxJQUFBO0lBQUEsU0FBQUEsSUFBQTtJQUFBLElBQUFDLGdCQUFBLENBQUFDLE9BQUEsK0JBQ2pCLENBQUM7SUFBQSxJQUFBRCxnQkFBQSxDQUFBQyxPQUFBLDBCQUNOLENBQUM7RUFBQTtFQUVuQkMsT0FBT0EsQ0FDVkMsTUFBd0IsRUFDeEJDLE9BQXlCLEVBQ3pCQyxVQUF3QyxFQUNqQztJQUNQLE1BQU1DLGFBQWEsR0FBR2xCLHFCQUFxQixDQUFDbUIsV0FBVyxDQUFDO0lBQ3hEO0lBQ0E7SUFDQTtJQUNBO0lBQ0E7SUFDQTtJQUNBLElBQUlELGFBQWEsS0FBSyxJQUFJLENBQUNFLG1CQUFtQixJQUFJLElBQUksQ0FBQ0EsbUJBQW1CLEtBQUssQ0FBQyxFQUFFO01BQzlFO01BQ0E7TUFDQSxNQUFNQyxRQUFRLEdBQUdOLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7O01BRTdCO01BQ0E7TUFDQTtNQUNBO01BQ0E7TUFDQSxNQUFNTyxNQUFNLEdBQUdwQixJQUFJLENBQUNxQixHQUFHLENBQUMsR0FBR0YsUUFBUSxDQUFDO01BQ3BDLE1BQU1HLE1BQU0sR0FBR3RCLElBQUksQ0FBQ3VCLEdBQUcsQ0FBQyxHQUFHSixRQUFRLENBQUM7TUFDcEMsTUFBTUssU0FBUyxHQUFHLElBQUFDLHFCQUFZLEVBQUNILE1BQU0sRUFBRSxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsR0FBRyxJQUFBRyxxQkFBWSxFQUFDTCxNQUFNLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDO01BRTNFLElBQUksQ0FBQ00sSUFBSSxDQUFDQyxXQUFXLENBQW9CO1FBQ3JDQyxFQUFFLEVBQUVDLG9CQUFZLENBQUNDLGFBQWE7UUFDOUJOLFNBQVMsRUFBRUEsU0FBUztRQUNwQk8sUUFBUSxFQUFFLElBQUksQ0FBQ0MsY0FBYztNQUNqQyxDQUFDLENBQUM7TUFDRixJQUFJLENBQUNkLG1CQUFtQixHQUFHZCxxQkFBcUIsQ0FBQ1ksYUFBYSxDQUFDO0lBQ25FOztJQUVBO0lBQ0EsSUFBSSxDQUFDVSxJQUFJLENBQUNDLFdBQVcsQ0FBaUI7TUFBRUMsRUFBRSxFQUFFQyxvQkFBWSxDQUFDSSxRQUFRO01BQUVDLFdBQVcsRUFBRWpCO0lBQVksQ0FBQyxDQUFDOztJQUU5RjtJQUNBO0lBQ0E7SUFDQSxPQUFPLElBQUk7RUFDZjtBQUNKO0FBRUFrQixpQkFBaUIsQ0FBQ0Msb0JBQVksRUFBRTlCLGNBQWMsQ0FBQztBQUFDLElBQUErQixRQUFBLEdBQUFDLE9BQUEsQ0FBQTNCLE9BQUEsR0FFakMsRUFBRSxFQUFFIiwiaWdub3JlTGlzdCI6W119