UNPKG

vmsg

Version:

Library for creating voice messages

758 lines (681 loc) 24.6 kB
/* eslint-disable */ function pad2(n) { n |= 0; return n < 10 ? `0${n}` : `${Math.min(n, 99)}`; } function inlineWorker() { // TODO(Kagami): Cache compiled module in IndexedDB? It works in FF // and Edge, see: https://github.com/mdn/webassembly-examples/issues/4 // Though gzipped WASM module currently weights ~70kb so it should be // perfectly cached by the browser itself. function fetchAndInstantiate(url, imports) { if (!WebAssembly.instantiateStreaming) return fetchAndInstantiateFallback(url, imports); const req = fetch(url, {credentials: "same-origin"}); return WebAssembly.instantiateStreaming(req, imports).catch(err => { // https://github.com/Kagami/vmsg/issues/11 if (err.message && err.message.indexOf("Argument 0 must be provided and must be a Response") > 0) { return fetchAndInstantiateFallback(url, imports); } else { throw err; } }); } function fetchAndInstantiateFallback(url, imports) { return new Promise((resolve, reject) => { const req = new XMLHttpRequest(); req.open("GET", url); req.responseType = "arraybuffer"; req.onload = () => { resolve(WebAssembly.instantiate(req.response, imports)); }; req.onerror = reject; req.send(); }); } // Must be in sync with emcc settings! const TOTAL_STACK = 5 * 1024 * 1024; const TOTAL_MEMORY = 16 * 1024 * 1024; const WASM_PAGE_SIZE = 64 * 1024; let memory = null; let dynamicTop = TOTAL_STACK; // TODO(Kagami): Grow memory? function sbrk(increment) { const oldDynamicTop = dynamicTop; dynamicTop += increment; return oldDynamicTop; } // TODO(Kagami): LAME calls exit(-1) on internal error. Would be nice // to provide custom DEBUGF/ERRORF for easier debugging. Currenty // those functions do nothing. function exit(status) { postMessage({type: "internal-error", data: status}); } let FFI = null; let ref = null; let pcm_l = null; function vmsg_init(rate) { ref = FFI.vmsg_init(rate); if (!ref) return false; const pcm_l_ref = new Uint32Array(memory.buffer, ref, 1)[0]; pcm_l = new Float32Array(memory.buffer, pcm_l_ref); return true; } function vmsg_encode(data) { pcm_l.set(data); return FFI.vmsg_encode(ref, data.length) >= 0; } function vmsg_flush() { if (FFI.vmsg_flush(ref) < 0) return null; const mp3_ref = new Uint32Array(memory.buffer, ref + 4, 1)[0]; const size = new Uint32Array(memory.buffer, ref + 8, 1)[0]; const mp3 = new Uint8Array(memory.buffer, mp3_ref, size); const blob = new Blob([mp3], {type: "audio/mpeg"}); FFI.vmsg_free(ref); ref = null; pcm_l = null; return blob; } // https://github.com/brion/min-wasm-fail function testSafariWebAssemblyBug() { const bin = new Uint8Array([0,97,115,109,1,0,0,0,1,6,1,96,1,127,1,127,3,2,1,0,5,3,1,0,1,7,8,1,4,116,101,115,116,0,0,10,16,1,14,0,32,0,65,1,54,2,0,32,0,40,2,0,11]); const mod = new WebAssembly.Module(bin); const inst = new WebAssembly.Instance(mod, {}); // test storing to and loading from a non-zero location via a parameter. // Safari on iOS 11.2.5 returns 0 unexpectedly at non-zero locations return (inst.exports.test(4) !== 0); } onmessage = (e) => { const msg = e.data; switch (msg.type) { case "init": const { wasmURL, shimURL } = msg.data; Promise.resolve().then(() => { if (self.WebAssembly && !testSafariWebAssemblyBug()) { delete self.WebAssembly; } if (!self.WebAssembly) { importScripts(shimURL); } memory = new WebAssembly.Memory({ initial: TOTAL_MEMORY / WASM_PAGE_SIZE, maximum: TOTAL_MEMORY / WASM_PAGE_SIZE, }); return { memory: memory, pow: Math.pow, exit: exit, powf: Math.pow, exp: Math.exp, sqrtf: Math.sqrt, cos: Math.cos, log: Math.log, sin: Math.sin, sbrk: sbrk, }; }).then(Runtime => { return fetchAndInstantiate(wasmURL, {env: Runtime}) }).then(wasm => { FFI = wasm.instance.exports; postMessage({type: "init", data: null}); }).catch(err => { postMessage({type: "init-error", data: err.toString()}); }); break; case "start": if (!vmsg_init(msg.data)) return postMessage({type: "error", data: "vmsg_init"}); break; case "data": if (!vmsg_encode(msg.data)) return postMessage({type: "error", data: "vmsg_encode"}); break; case "stop": const blob = vmsg_flush(); if (!blob) return postMessage({type: "error", data: "vmsg_flush"}); postMessage({type: "stop", data: blob}); break; } }; } export class Recorder { constructor(opts = {}, onStop = null) { // Can't use relative URL in blob worker, see: // https://stackoverflow.com/a/22582695 this.wasmURL = new URL(opts.wasmURL || "/static/js/vmsg.wasm", location).href; this.shimURL = new URL(opts.shimURL || "/static/js/wasm-polyfill.js", location).href; this.onStop = onStop; this.pitch = opts.pitch || 0; this.stream = null; this.audioCtx = null; this.gainNode = null; this.pitchFX = null; this.encNode = null; this.worker = null; this.workerURL = null; this.blob = null; this.blobURL = null; this.resolve = null; this.reject = null; Object.seal(this); } close() { if (this.encNode) this.encNode.disconnect(); if (this.encNode) this.encNode.onaudioprocess = null; if (this.stream) this.stopTracks(); if (this.audioCtx) this.audioCtx.close(); if (this.worker) { this.worker.terminate(); this.worker = null; } if (this.workerURL) URL.revokeObjectURL(this.workerURL); if (this.blobURL) URL.revokeObjectURL(this.blobURL); } // Without pitch shift: // [sourceNode] -> [gainNode] -> [encNode] -> [audioCtx.destination] // | // -> [worker] // With pitch shift: // [sourceNode] -> [gainNode] -> [pitchFX] -> [encNode] -> [audioCtx.destination] // | // -> [worker] initAudio() { const getUserMedia = navigator.mediaDevices && navigator.mediaDevices.getUserMedia ? function(constraints) { return navigator.mediaDevices.getUserMedia(constraints); } : function(constraints) { const oldGetUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia; if (!oldGetUserMedia) { return Promise.reject(new Error("getUserMedia is not implemented in this browser")); } return new Promise(function(resolve, reject) { oldGetUserMedia.call(navigator, constraints, resolve, reject); }); }; return getUserMedia({audio: true}).then((stream) => { this.stream = stream; const audioCtx = this.audioCtx = new (window.AudioContext || window.webkitAudioContext)(); const sourceNode = audioCtx.createMediaStreamSource(stream); const gainNode = this.gainNode = (audioCtx.createGain || audioCtx.createGainNode).call(audioCtx); gainNode.gain.value = 1; sourceNode.connect(gainNode); const pitchFX = this.pitchFX = new Jungle(audioCtx); pitchFX.setPitchOffset(this.pitch); const encNode = this.encNode = (audioCtx.createScriptProcessor || audioCtx.createJavaScriptNode).call(audioCtx, 0, 1, 1); pitchFX.output.connect(encNode); gainNode.connect(this.pitch === 0 ? encNode : pitchFX.input); }); } initWorker() { if (this.worker) return Promise.resolve(); // https://stackoverflow.com/a/19201292 const blob = new Blob( ["(", inlineWorker.toString(), ")()"], {type: "application/javascript"}); const workerURL = this.workerURL = URL.createObjectURL(blob); const worker = this.worker = new Worker(workerURL); const { wasmURL, shimURL } = this; worker.postMessage({type: "init", data: {wasmURL, shimURL}}); return new Promise((resolve, reject) => { worker.onmessage = (e) => { const msg = e.data; switch (msg.type) { case "init": resolve(); break; case "init-error": this.close(); reject(new Error(msg.data)); break; // TODO(Kagami): Error handling. case "error": case "internal-error": this.close(); console.error("Worker error:", msg.data); if (this.reject) this.reject(msg.data); break; case "stop": this.blob = msg.data; this.blobURL = URL.createObjectURL(msg.data); if (this.onStop) this.onStop(); if (this.resolve) this.resolve(this.blob); break; } } }); } init() { return this.initAudio().then(this.initWorker.bind(this)); } startRecording() { if (!this.stream) throw new Error("missing audio initialization"); if (!this.worker) throw new Error("missing worker initialization"); this.blob = null; if (this.blobURL) URL.revokeObjectURL(this.blobURL); this.blobURL = null; this.resolve = null; this.reject = null; this.worker.postMessage({type: "start", data: this.audioCtx.sampleRate}); this.encNode.onaudioprocess = (e) => { const samples = e.inputBuffer.getChannelData(0); this.worker.postMessage({type: "data", data: samples}); }; this.encNode.connect(this.audioCtx.destination); } stopRecording() { if (!this.stream) throw new Error("missing audio initialization"); if (!this.worker) throw new Error("missing worker initialization"); this.encNode.disconnect(); this.encNode.onaudioprocess = null; this.stopTracks(); this.audioCtx.close(); this.worker.postMessage({type: "stop", data: null}); return new Promise((resolve, reject) => { this.resolve = resolve; this.reject = reject; }); } stopTracks() { // Might be missed in Safari and old FF/Chrome per MDN. if (this.stream.getTracks) { // Hide browser's recording indicator. this.stream.getTracks().forEach((track) => track.stop()); } } } export class Form { constructor(opts = {}, resolve, reject) { this.recorder = new Recorder(opts, this.onStop.bind(this)); this.resolve = resolve; this.reject = reject; this.backdrop = null; this.popup = null; this.recordBtn = null; this.stopBtn = null; this.timer = null; this.audio = null; this.saveBtn = null; this.tid = 0; this.start = 0; Object.seal(this); this.recorder.initAudio() .then(() => this.drawInit()) .then(() => this.recorder.initWorker()) .then(() => this.drawAll()) .catch((err) => this.drawError(err)); } drawInit() { if (this.backdrop) return; const backdrop = this.backdrop = document.createElement("div"); backdrop.className = "vmsg-backdrop"; backdrop.addEventListener("click", () => this.close(null)); const popup = this.popup = document.createElement("div"); popup.className = "vmsg-popup"; popup.addEventListener("click", (e) => e.stopPropagation()); const progress = document.createElement("div"); progress.className = "vmsg-progress"; for (let i = 0; i < 3; i++) { const progressDot = document.createElement("div"); progressDot.className = "vmsg-progress-dot"; progress.appendChild(progressDot); } popup.appendChild(progress); backdrop.appendChild(popup); document.body.appendChild(backdrop); } drawTime(msecs) { const secs = Math.round(msecs / 1000); this.timer.textContent = pad2(secs / 60) + ":" + pad2(secs % 60); } drawAll() { this.drawInit(); this.clearAll(); const recordRow = document.createElement("div"); recordRow.className = "vmsg-record-row"; this.popup.appendChild(recordRow); const recordBtn = this.recordBtn = document.createElement("button"); recordBtn.className = "vmsg-button vmsg-record-button"; recordBtn.textContent = "●"; recordBtn.title = "Start Recording"; recordBtn.addEventListener("click", () => this.startRecording()); recordRow.appendChild(recordBtn); const stopBtn = this.stopBtn = document.createElement("button"); stopBtn.className = "vmsg-button vmsg-stop-button"; stopBtn.style.display = "none"; stopBtn.textContent = "■"; stopBtn.title = "Stop Recording"; stopBtn.addEventListener("click", () => this.stopRecording()); recordRow.appendChild(stopBtn); const audio = this.audio = new Audio(); audio.autoplay = true; const timer = this.timer = document.createElement("span"); timer.className = "vmsg-timer"; timer.title = "Preview Recording"; timer.addEventListener("click", () => { if (audio.paused) { if (this.recorder.blobURL) { audio.src = this.recorder.blobURL; } } else { audio.pause(); } }); this.drawTime(0); recordRow.appendChild(timer); const saveBtn = this.saveBtn = document.createElement("button"); saveBtn.className = "vmsg-button vmsg-save-button"; saveBtn.textContent = "✓"; saveBtn.title = "Save Recording"; saveBtn.disabled = true; saveBtn.addEventListener("click", () => this.close(this.recorder.blob)); recordRow.appendChild(saveBtn); const gainWrapper = document.createElement("div"); gainWrapper.className = "vmsg-slider-wrapper vmsg-gain-slider-wrapper"; const gainSlider = document.createElement("input"); gainSlider.className = "vmsg-slider vmsg-gain-slider"; gainSlider.setAttribute("type", "range"); gainSlider.min = 0; gainSlider.max = 2; gainSlider.step = 0.2; gainSlider.value = 1; gainSlider.onchange = () => { const gain = +gainSlider.value; this.recorder.gainNode.gain.value = gain; }; gainWrapper.appendChild(gainSlider); this.popup.appendChild(gainWrapper); const pitchWrapper = document.createElement("div"); pitchWrapper.className = "vmsg-slider-wrapper vmsg-pitch-slider-wrapper"; const pitchSlider = document.createElement("input"); pitchSlider.className = "vmsg-slider vmsg-pitch-slider"; pitchSlider.setAttribute("type", "range"); pitchSlider.min = -1; pitchSlider.max = 1; pitchSlider.step = 0.2; pitchSlider.value = this.recorder.pitch; pitchSlider.onchange = () => { const pitch = +pitchSlider.value; this.recorder.pitchFX.setPitchOffset(pitch); this.recorder.gainNode.disconnect(); this.recorder.gainNode.connect( pitch === 0 ? this.recorder.encNode : this.recorder.pitchFX.input ); }; pitchWrapper.appendChild(pitchSlider); this.popup.appendChild(pitchWrapper); recordBtn.focus(); } drawError(err) { console.error(err); this.drawInit(); this.clearAll(); const error = document.createElement("div"); error.className = "vmsg-error"; error.textContent = err.toString(); this.popup.appendChild(error); } clearAll() { if (!this.popup) return; this.popup.innerHTML = ""; } close(blob) { if (this.audio) this.audio.pause(); if (this.tid) clearTimeout(this.tid); this.recorder.close(); this.backdrop.remove(); if (blob) { this.resolve(blob); } else { this.reject(new Error("No record made")); } } onStop() { this.recordBtn.style.display = ""; this.stopBtn.style.display = "none"; this.stopBtn.disabled = false; this.saveBtn.disabled = false; } startRecording() { this.audio.pause(); this.start = Date.now(); this.updateTime(); this.recordBtn.style.display = "none"; this.stopBtn.style.display = ""; this.saveBtn.disabled = true; this.stopBtn.focus(); this.recorder.startRecording(); } stopRecording() { clearTimeout(this.tid); this.tid = 0; this.stopBtn.disabled = true; this.recordBtn.focus(); this.recorder.stopRecording(); } updateTime() { // NOTE(Kagami): We can do this in `onaudioprocess` but that would // run too often and create unnecessary DOM updates. this.drawTime(Date.now() - this.start); this.tid = setTimeout(() => this.updateTime(), 300); } } let shown = false; /** * Record a new voice message. * * @param {Object=} opts - Options * @param {string=} opts.wasmURL - URL of the module * ("/static/js/vmsg.wasm" by default) * @param {string=} opts.shimURL - URL of the WebAssembly polyfill * ("/static/js/wasm-polyfill.js" by default) * @param {number=} opts.pitch - Initial pitch shift ([-1, 1], 0 by default) * @return {Promise.<Blob>} A promise that contains recorded blob when fulfilled. */ export function record(opts) { return new Promise((resolve, reject) => { if (shown) throw new Error("Record form is already opened"); shown = true; new Form(opts, resolve, reject); // Use `.finally` once it's available in Safari and Edge. }).then(result => { shown = false; return result; }, err => { shown = false; throw err; }); } /** * All available public items. */ export default { Recorder, Form, record }; // Borrowed from and slightly modified: // https://github.com/cwilso/Audio-Input-Effects/blob/master/js/jungle.js // Copyright 2012, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. const delayTime = 0.100; const fadeTime = 0.050; const bufferTime = 0.100; function createFadeBuffer(context, activeTime, fadeTime) { var length1 = activeTime * context.sampleRate; var length2 = (activeTime - 2*fadeTime) * context.sampleRate; var length = length1 + length2; var buffer = context.createBuffer(1, length, context.sampleRate); var p = buffer.getChannelData(0); var fadeLength = fadeTime * context.sampleRate; var fadeIndex1 = fadeLength; var fadeIndex2 = length1 - fadeLength; // 1st part of cycle for (var i = 0; i < length1; ++i) { var value; if (i < fadeIndex1) { value = Math.sqrt(i / fadeLength); } else if (i >= fadeIndex2) { value = Math.sqrt(1 - (i - fadeIndex2) / fadeLength); } else { value = 1; } p[i] = value; } // 2nd part for (var i = length1; i < length; ++i) { p[i] = 0; } return buffer; } function createDelayTimeBuffer(context, activeTime, fadeTime, shiftUp) { var length1 = activeTime * context.sampleRate; var length2 = (activeTime - 2*fadeTime) * context.sampleRate; var length = length1 + length2; var buffer = context.createBuffer(1, length, context.sampleRate); var p = buffer.getChannelData(0); // 1st part of cycle for (var i = 0; i < length1; ++i) { if (shiftUp) // This line does shift-up transpose p[i] = (length1-i)/length; else // This line does shift-down transpose p[i] = i / length1; } // 2nd part for (var i = length1; i < length; ++i) { p[i] = 0; } return buffer; } function Jungle(context) { this.context = context; // Create nodes for the input and output of this "module". var input = (context.createGain || context.createGainNode).call(context); var output = (context.createGain || context.createGainNode).call(context); this.input = input; this.output = output; // Delay modulation. var mod1 = context.createBufferSource(); var mod2 = context.createBufferSource(); var mod3 = context.createBufferSource(); var mod4 = context.createBufferSource(); this.shiftDownBuffer = createDelayTimeBuffer(context, bufferTime, fadeTime, false); this.shiftUpBuffer = createDelayTimeBuffer(context, bufferTime, fadeTime, true); mod1.buffer = this.shiftDownBuffer; mod2.buffer = this.shiftDownBuffer; mod3.buffer = this.shiftUpBuffer; mod4.buffer = this.shiftUpBuffer; mod1.loop = true; mod2.loop = true; mod3.loop = true; mod4.loop = true; // for switching between oct-up and oct-down var mod1Gain = (context.createGain || context.createGainNode).call(context); var mod2Gain = (context.createGain || context.createGainNode).call(context); var mod3Gain = (context.createGain || context.createGainNode).call(context); mod3Gain.gain.value = 0; var mod4Gain = (context.createGain || context.createGainNode).call(context); mod4Gain.gain.value = 0; mod1.connect(mod1Gain); mod2.connect(mod2Gain); mod3.connect(mod3Gain); mod4.connect(mod4Gain); // Delay amount for changing pitch. var modGain1 = (context.createGain || context.createGainNode).call(context); var modGain2 = (context.createGain || context.createGainNode).call(context); var delay1 = (context.createDelay || context.createDelayNode).call(context); var delay2 = (context.createDelay || context.createDelayNode).call(context); mod1Gain.connect(modGain1); mod2Gain.connect(modGain2); mod3Gain.connect(modGain1); mod4Gain.connect(modGain2); modGain1.connect(delay1.delayTime); modGain2.connect(delay2.delayTime); // Crossfading. var fade1 = context.createBufferSource(); var fade2 = context.createBufferSource(); var fadeBuffer = createFadeBuffer(context, bufferTime, fadeTime); fade1.buffer = fadeBuffer fade2.buffer = fadeBuffer; fade1.loop = true; fade2.loop = true; var mix1 = (context.createGain || context.createGainNode).call(context); var mix2 = (context.createGain || context.createGainNode).call(context); mix1.gain.value = 0; mix2.gain.value = 0; fade1.connect(mix1.gain); fade2.connect(mix2.gain); // Connect processing graph. input.connect(delay1); input.connect(delay2); delay1.connect(mix1); delay2.connect(mix2); mix1.connect(output); mix2.connect(output); // Start var t = context.currentTime + 0.050; var t2 = t + bufferTime - fadeTime; mod1.start(t); mod2.start(t2); mod3.start(t); mod4.start(t2); fade1.start(t); fade2.start(t2); this.mod1 = mod1; this.mod2 = mod2; this.mod1Gain = mod1Gain; this.mod2Gain = mod2Gain; this.mod3Gain = mod3Gain; this.mod4Gain = mod4Gain; this.modGain1 = modGain1; this.modGain2 = modGain2; this.fade1 = fade1; this.fade2 = fade2; this.mix1 = mix1; this.mix2 = mix2; this.delay1 = delay1; this.delay2 = delay2; this.setDelay(delayTime); } Jungle.prototype.setDelay = function(delayTime) { this.modGain1.gain.setTargetAtTime(0.5*delayTime, 0, 0.010); this.modGain2.gain.setTargetAtTime(0.5*delayTime, 0, 0.010); }; Jungle.prototype.setPitchOffset = function(mult) { if (mult>0) { // pitch up this.mod1Gain.gain.value = 0; this.mod2Gain.gain.value = 0; this.mod3Gain.gain.value = 1; this.mod4Gain.gain.value = 1; } else { // pitch down this.mod1Gain.gain.value = 1; this.mod2Gain.gain.value = 1; this.mod3Gain.gain.value = 0; this.mod4Gain.gain.value = 0; } this.setDelay(delayTime*Math.abs(mult)); };