@itslanguage/api

<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8"> <title>utils/audio-over-socket.js - ITSLanguage JavaScript SDK</title> <meta name="description" content="Speech technology for language education. JSDoc 3 website for the JavaScript SDK." /> <meta name="keywords" content="itslangauge, jsdoc, documentation, speech, speech technology" /> <meta name="keyword" content="itslangauge, jsdoc, documentation, speech, speech technology" /> <meta property="og:title" content="ITSLanguage JavaScript SDK"/> <meta property="og:type" content="website"/> <meta property="og:image" content="https://avatars0.githubusercontent.com/u/20972344?s=400&u=73936e7fa2d2a1e5c180a11675bf9166d3717a6d&v=4"/> <meta property="og:site_name" content="ITSLanguage JavaScript SDK"/> <meta property="og:url" content="http://itslanguage.github.io/itslanguage-js/"/> <script src="scripts/prettify/prettify.js"></script> <script src="scripts/prettify/lang-css.js"></script>  <link type="text/css" rel="stylesheet" href="styles/prettify.css"> <link type="text/css" rel="stylesheet" href="styles/jsdoc.css"> <script src="scripts/nav.js" defer></script> <meta name="viewport" content="width=device-width, initial-scale=1.0"> </head> <body> <input type="checkbox" id="nav-trigger" class="nav-trigger" /> <label for="nav-trigger" class="navicon-button x"> <div class="navicon"></div> </label> <label for="nav-trigger" class="overlay"></label> <nav > <input type="text" id="nav-search" placeholder="Search" /> <h2><a href="index.html">Home</a></h2><h2><a href="https://github.com/itslanguage/itslanguage-js" target="_blank" class="menu-item" id="repository" >Github repo</a></h2><h3>Modules</h3><ul><li><a href="module-api.html">api</a><ul class='members'><li data-type='member'><a href="module-api.html#.version">version</a></li></ul><ul class='methods'><li data-type='method'><a href="module-api.html#.createItslApi">createItslApi</a></li></ul></li><li><a href="module-api_authentication.html">api/authentication</a><ul class='methods'><li data-type='method'><a href="module-api_authentication.html#.assembleScope">assembleScope</a></li><li data-type='method'><a href="module-api_authentication.html#.authenticate">authenticate</a></li><li data-type='method'><a href="module-api_authentication.html#.impersonate">impersonate</a></li></ul></li><li><a href="module-api_basicauth.html">api/basicauth</a><ul class='methods'><li data-type='method'><a href="module-api_basicauth.html#.create">create</a></li></ul></li><li><a href="module-api_broadcaster.html">api/broadcaster</a></li><li><a href="module-api_categories.html">api/categories</a><ul class='methods'><li data-type='method'><a href="module-api_categories.html#.create">create</a></li><li data-type='method'><a href="module-api_categories.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_categories.html#.getAllWithParentId">getAllWithParentId</a></li><li data-type='method'><a href="module-api_categories.html#.getById">getById</a></li><li data-type='method'><a href="module-api_categories.html#.update">update</a></li></ul></li><li><a href="module-api_challenges.html">api/challenges</a><ul class='members'><li data-type='member'><a href="module-api_challenges.html#.choice">choice</a></li><li data-type='member'><a href="module-api_challenges.html#.choiceRecognition">choiceRecognition</a></li><li data-type='member'><a href="module-api_challenges.html#.feedback">feedback</a></li><li data-type='member'><a href="module-api_challenges.html#.feedbackSpeech">feedbackSpeech</a></li><li data-type='member'><a href="module-api_challenges.html#.pronunciation">pronunciation</a></li><li data-type='member'><a href="module-api_challenges.html#.pronunciationAnalysis">pronunciationAnalysis</a></li><li data-type='member'><a href="module-api_challenges.html#.speech">speech</a></li><li data-type='member'><a href="module-api_challenges.html#.speechRecording">speechRecording</a></li></ul></li><li><a href="module-api_challenges_choice.html">api/challenges/choice</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_choice.html#.create">create</a></li><li data-type='method'><a href="module-api_challenges_choice.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_challenges_choice.html#.getById">getById</a></li></ul></li><li><a href="module-api_challenges_choice_recognition.html">api/challenges/choice/recognition</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.create">create</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.getById">getById</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.prepare">prepare</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.prepareChallenge">prepareChallenge</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.recognise">recognise</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.recogniseAudioStream">recogniseAudioStream</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#.recogniseNonStreaming">recogniseNonStreaming</a></li><li data-type='method'><a href="module-api_challenges_choice_recognition.html#~url">url</a></li></ul></li><li><a href="module-api_challenges_feedback.html">api/challenges/feedback</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_feedback.html#.create">create</a></li><li data-type='method'><a href="module-api_challenges_feedback.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_challenges_feedback.html#.getById">getById</a></li></ul></li><li><a href="module-api_challenges_feedback_speech.html">api/challenges/feedback/speech</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_feedback_speech.html#.feedback">feedback</a></li><li data-type='method'><a href="module-api_challenges_feedback_speech.html#.listenAndReply">listenAndReply</a></li><li data-type='method'><a href="module-api_challenges_feedback_speech.html#.pause">pause</a></li><li data-type='method'><a href="module-api_challenges_feedback_speech.html#.prepare">prepare</a></li><li data-type='method'><a href="module-api_challenges_feedback_speech.html#.resume">resume</a></li></ul></li><li><a href="module-api_challenges_pronunciation.html">api/challenges/pronunciation</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_pronunciation.html#.create">create</a></li><li data-type='method'><a href="module-api_challenges_pronunciation.html#.deleteChallenge">deleteChallenge</a></li><li data-type='method'><a href="module-api_challenges_pronunciation.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_challenges_pronunciation.html#.getById">getById</a></li></ul></li><li><a href="module-api_challenges_pronunciation_analysis.html">api/challenges/pronunciation/analysis</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.alignChallenge">alignChallenge</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.endStreamAudio">endStreamAudio</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.getById">getById</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.prepare">prepare</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.prepareAudio">prepareAudio</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.prepareChallenge">prepareChallenge</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#.streamAudio">streamAudio</a></li><li data-type='method'><a href="module-api_challenges_pronunciation_analysis.html#~url">url</a></li></ul></li><li><a href="module-api_challenges_speech.html">api/challenges/speech</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_speech.html#.create">create</a></li><li data-type='method'><a href="module-api_challenges_speech.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_challenges_speech.html#.getById">getById</a></li></ul></li><li><a href="module-api_challenges_speech_recordings.html">api/challenges/speech/recordings</a><ul class='methods'><li data-type='method'><a href="module-api_challenges_speech_recordings.html#.create">create</a></li><li data-type='method'><a href="module-api_challenges_speech_recordings.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_challenges_speech_recordings.html#.getById">getById</a></li><li data-type='method'><a href="module-api_challenges_speech_recordings.html#.record">record</a></li><li data-type='method'><a href="module-api_challenges_speech_recordings.html#~url">url</a></li></ul></li><li><a href="module-api_communication.html">api/communication</a><ul class='members'><li data-type='member'><a href="module-api_communication.html#.settings">settings</a></li><li data-type='member'><a href="module-api_communication.html#~APPLICATION_JSON">APPLICATION_JSON</a></li><li data-type='member'><a href="module-api_communication.html#~AUTHORIZATION">AUTHORIZATION</a></li><li data-type='member'><a href="module-api_communication.html#~CONTENT_TYPE">CONTENT_TYPE</a></li><li data-type='member'><a href="module-api_communication.html#~LINK">LINK</a></li></ul><ul class='methods'><li data-type='method'><a href="module-api_communication.html#.addAccessToken">addAccessToken</a></li><li data-type='method'><a href="module-api_communication.html#.authorisedRequest">authorisedRequest</a></li><li data-type='method'><a href="module-api_communication.html#.request">request</a></li><li data-type='method'><a href="module-api_communication.html#.updateSettings">updateSettings</a></li><li data-type='method'><a href="module-api_communication.html#~getBearerToken">getBearerToken</a></li><li data-type='method'><a href="module-api_communication.html#~handleResponse">handleResponse</a></li></ul></li><li><a href="module-api_communication_websocket.html">api/communication/websocket</a><ul class='methods'><li data-type='method'><a href="module-api_communication_websocket.html#.closeWebsocketConnection">closeWebsocketConnection</a></li><li data-type='method'><a href="module-api_communication_websocket.html#.getWebsocketConnection">getWebsocketConnection</a></li><li data-type='method'><a href="module-api_communication_websocket.html#.makeWebsocketCall">makeWebsocketCall</a></li><li data-type='method'><a href="module-api_communication_websocket.html#.openWebsocketConnection">openWebsocketConnection</a></li><li data-type='method'><a href="module-api_communication_websocket.html#~establishNewBundesbahn">establishNewBundesbahn</a></li><li data-type='method'><a href="module-api_communication_websocket.html#~handleWebsocketAuthorisationChallenge">handleWebsocketAuthorisationChallenge</a></li></ul></li><li><a href="module-api_emailauth.html">api/emailauth</a><ul class='methods'><li data-type='method'><a href="module-api_emailauth.html#.create">create</a></li><li data-type='method'><a href="module-api_emailauth.html#~url">url</a></li></ul></li><li><a href="module-api_groups.html">api/groups</a><ul class='methods'><li data-type='method'><a href="module-api_groups.html#.create">create</a></li><li data-type='method'><a href="module-api_groups.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_groups.html#.getById">getById</a></li></ul></li><li><a href="module-api_organisations.html">api/organisations</a><ul class='methods'><li data-type='method'><a href="module-api_organisations.html#.create">create</a></li><li data-type='method'><a href="module-api_organisations.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_organisations.html#.getById">getById</a></li></ul></li><li><a href="module-api_profile.html">api/profile</a><ul class='methods'><li data-type='method'><a href="module-api_profile.html#.create">create</a></li><li data-type='method'><a href="module-api_profile.html#.getById">getById</a></li><li data-type='method'><a href="module-api_profile.html#.getCurrent">getCurrent</a></li><li data-type='method'><a href="module-api_profile.html#~url">url</a></li></ul></li><li><a href="module-api_progress.html">api/progress</a><ul class='methods'><li data-type='method'><a href="module-api_progress.html#.getById">getById</a></li></ul></li><li><a href="module-api_roles.html">api/roles</a><ul class='methods'><li data-type='method'><a href="module-api_roles.html#.create">create</a></li><li data-type='method'><a href="module-api_roles.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_roles.html#.getById">getById</a></li></ul></li><li><a href="module-api_tenants.html">api/tenants</a><ul class='methods'><li data-type='method'><a href="module-api_tenants.html#.create">create</a></li><li data-type='method'><a href="module-api_tenants.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_tenants.html#.getById">getById</a></li></ul></li><li><a href="module-api_users.html">api/users</a><ul class='methods'><li data-type='method'><a href="module-api_users.html#.create">create</a></li><li data-type='method'><a href="module-api_users.html#.getAll">getAll</a></li><li data-type='method'><a href="module-api_users.html#.getById">getById</a></li><li data-type='method'><a href="module-api_users.html#.getCurrent">getCurrent</a></li></ul></li><li><a href="module-api_utils.html">api/utils</a><ul class='methods'><li data-type='method'><a href="module-api_utils.html#.asyncBlobToArrayBuffer">asyncBlobToArrayBuffer</a></li><li data-type='method'><a href="module-api_utils.html#.checkAudioIsNotEmpty">checkAudioIsNotEmpty</a></li><li data-type='method'><a href="module-api_utils.html#.dataToBase64">dataToBase64</a></li></ul></li><li><a href="module-api_utils_audio-over-socket.html">api/utils/audio-over-socket</a><ul class='methods'><li data-type='method'><a href="module-api_utils_audio-over-socket.html#.encodeAndSendAudioOnDataAvailable">encodeAndSendAudioOnDataAvailable</a></li><li data-type='method'><a href="module-api_utils_audio-over-socket.html#.prepareServerForAudio">prepareServerForAudio</a></li><li data-type='method'><a href="module-api_utils_audio-over-socket.html#.registerStreamForRecorder">registerStreamForRecorder</a></li></ul></li></ul><h3>Classes</h3><ul><li><a href="module-api.Itslanguage.html">Itslanguage</a></li></ul><h3>Events</h3><ul><li><a href="broadcaster.html#event:websocketserverreadyforaudio">websocketserverreadyforaudio</a></li></ul> </nav> <div id="main"> <h1 class="page-title">utils/audio-over-socket.js</h1> <section> <article> <pre class="prettyprint source linenums"><code>/** * This file contains some re-usable parts for websocket audio communication. * * @module api/utils/audio-over-socket */ import autobahn from 'autobahn'; import { getWebsocketConnection, makeWebsocketCall, } from '../communication/websocket'; import broadcaster from '../broadcaster'; import { dataToBase64, asyncBlobToArrayBuffer, checkAudioIsNotEmpty, } from './index'; /** * This class allows us to stream audio from the recorder to the backend. * @private */ class StreamRecorderAudio { /** * @param {MediaRecorder} recorder - Recorder to use to capture data from. * @param {string} rpcName - Name of the registered RPC function. * @param {object} websocketConnection - The websocket connection to use. * @param {string} [dataEvent=dataavailable] - Optional, the name of the event * to receive audio data on. Defaults to `dataavailable` but can be overridden * (for example when you want to use the BufferPlugin`. */ constructor( recorder, rpcName, websocketConnection, dataEvent = 'dataavailable', ) { /** * MediaRecorder to process the stream from. * @type {MediaRecorder} */ this.recorder = recorder; /** * Name of the RPC registered. * This name will be prepended with 'nl.itslanguage' for better consistency. * @type {string} */ this.rpcName = `nl.itslanguage.${rpcName}`; /** * Store a reference to the websocket connection. * @type {autobahn.Connection} */ this.websocketConnection = websocketConnection; this.dataEvent = dataEvent; /** * The autobahn.Registration object. This is returned when you register * a function through Session.register. * @type {null|autobahn.Registration} */ this.registration = null; this.sendAudioChunks = this.sendAudioChunks.bind(this); this.register = this.register.bind(this); this.unregister = this.unregister.bind(this); } /** * This is the function that will be registered to the autobahn realm that the backend will call * to receive audio on. * * Once called, it will prepare the recorder to allow data transmission trough the progressive * results meganism. * * @see https://github.com/crossbario/autobahn-js/blob/master/doc/reference.md#register * @see https://github.com/crossbario/autobahn-js/blob/master/doc/reference.md#progressive-results * * @private * @param {Array} args - Argument list. * @param {Object} kwargs - Key-valued argument list. * @param {Object} details - Details, just as the progress function. * @returns {Promise} - A promise that can be resolved to end the asynchronous behaviour of this * registered RCP. */ sendAudioChunks(args, kwargs, details) { const defer = new autobahn.when.defer(); // eslint-disable-line new-cap let lastChunk = false; let audioSent = false; const resolve = () => { defer.resolve(); lastChunk = false; audioSent = false; this.unregister(); }; const processData = ({ data }) => { if (checkAudioIsNotEmpty(data.size, this.recorder.mimeType)) { asyncBlobToArrayBuffer(data).then(audioData => { if (details.progress) { const dataToSend = Array.from(new Uint8Array(audioData)); details.progress([dataToSend]); audioSent = true; // If the last one ends, closing time! if (lastChunk) { resolve(); } } else { defer.reject('no progress function registered'); } }); } }; const recorderStopped = () => { // If we call stop without sending data we will resolve the autobahn rpc. if (!audioSent) { resolve(); } else { // When stopped, the dataavailableevent will be triggered // one final time, so make sure it will cleanup afterwards lastChunk = true; } // Recording done; clean up!; this.recorder.removeEventListener(this.dataEvent, processData); this.recorder.removeEventListener('stop', recorderStopped); }; // Now add the event listeners! this.recorder.addEventListener(this.dataEvent, processData); this.recorder.addEventListener('stop', recorderStopped); // Notify listeners that we are ready to process audio; this.recorder.dispatchEvent(new Event('recorderready')); return defer.promise; } /** * register the RPC to the autobahn realm. * @returns {Promise} */ register() { return new Promise((resolve, reject) => { const { session } = this.websocketConnection; // First cleanup previously created registrations on this session; Promise.all( session.registrations.map(reg => session.unregister(reg)), ).then(() => { session .register(this.rpcName, this.sendAudioChunks) .then(registration => { this.registration = registration; resolve(registration); }) .catch(reject); }); }); } /** * unregister the RPC from the autobahn realm. */ unregister() { return new Promise((resolve, reject) => { /* istanbul ignore if */ if (!this.registration) { // Because the unregister method is hidden by the private StreamAudio // class it is impossible to test this path right now. resolve(); // There is no registration to unregister! } else { this.websocketConnection.session .unregister(this.registration) .then(() => { this.registration = null; resolve(); }) .catch(reject); } }); } } /** * Register a RPC call to the current websocket connection. The backend will call this registered * function once, an then we can send progressive results (the details.progress call) to send audio * chunks to the backend. We will send those chunks as soon as we got audio from the recorder. * * When the recording ends we un-register the rpc. * * @param {MediaRecorder} recorder - Audio recorder instance. * @param {string} rpcName - Name of the RPC to register. This name will be prepended with * nl.itslanguage for better consistency. * @fires broadcaster#websocketserverreadyforaudio * @param {string} [dataEven] - Optional, the name of the event to receive audio * data on. Can be overridden, for example when you want to use the * `BufferPlugin`. * @returns {Promise} - It returns a promise with the service registration as result. */ export function registerStreamForRecorder(recorder, rpcName, dataEvent) { // Start registering a RPC call. As a result, this function will return a promise with the // registration of the RPC as result. return new Promise((resolve, reject) => { getWebsocketConnection().then(websocketConnection => { const streamingSession = new StreamRecorderAudio( recorder, rpcName, websocketConnection, dataEvent, ); streamingSession .register() .then(registration => { /** * Notify that we are ready to process audio. * @event broadcaster#websocketserverreadyforaudio * @deprecated will be removed in a future version */ broadcaster.emit('websocketserverreadyforaudio'); recorder.dispatchEvent(new Event('websocketserverreadyforaudio')); resolve(registration); }) .catch(reject); }); }); } /** * Encode the audio as base64 and send it to the websocket server. * * @param {string} id - The reserved ID for the audio. * @param {MediaRecorder} recorder - The recorder to use to get the recording. * @param {string} rpc - The RPC to use to store the data. * * @returns {Promise<*>} - The response of the given RPC. */ export function encodeAndSendAudioOnDataAvailable(id, recorder, rpc) { return new Promise((resolve, reject) => { let lastChunk = false; // When data is received from the recorder, it will be in Blob format. // When we read the data from the Blob element, base64 it and send it to // the websocket server and continue with the chain. const processData = ({ data }) => { asyncBlobToArrayBuffer(data).then(audioData => { const encoded = dataToBase64(audioData); // Send the audio makeWebsocketCall(rpc, { args: [id, encoded, 'base64'] }) .then(result => { /* istanbul ignore else */ if (lastChunk) { lastChunk = false; resolve(result); } }) .catch(error => { reject(error); }); }); }; const recorderStopped = () => { // When stopped, the dataavailable event will be triggered // one final time, so make sure it will cleanup afterwards lastChunk = true; // Recording done, self cleanup! recorder.removeEventListener('dataavailable', processData); recorder.removeEventListener('stop', recorderStopped); }; // Now add the event listeners! recorder.addEventListener('dataavailable', processData); recorder.addEventListener('stop', recorderStopped); // Notify listeners that we are ready to process audio; recorder.dispatchEvent(new Event('recorderready')); }); } /** * Send the recorder settings to the websocket server to initialize it. * * The reserved ID (passed in the parameters) is returned once the promise is resolved. * * @param {string} id - The reserved ID for the audio. * @param {MediaRecorder} recorder - The recorder which has been set up to record. * @param {string} rpc - The RPC to use to initialize the websocket server. * * @emits {websocketserverreadyforaudio} - When the websocket server has been prepared for and is * ready to receive the audio. * * @returns {Promise} - The promise which resolves when the websocket server is ready for the audio. */ export function prepareServerForAudio(id, recorder, rpc) { const { audioFormat, audioParameters } = recorder.getAudioSpecs(); return makeWebsocketCall(rpc, { args: [id, audioFormat], kwargs: audioParameters, }).then(() => { // We've prepped the websocket server, now it can receive audio. Broadcast // that it is allowed to record. // This call is deprecated and will be removed in a future version, the // event on the recorder will stay. broadcaster.emit('websocketserverreadyforaudio'); recorder.dispatchEvent(new Event('websocketserverreadyforaudio')); return id; }); } </code></pre> </article> </section> </div> <br class="clear"> <footer> Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.3</a> on Mon Oct 12 2020 12:26:34 GMT+0000 (Coordinated Universal Time) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme. </footer> <script>prettyPrint();</script> <script src="scripts/polyfill.js"></script> <script src="scripts/linenumber.js"></script> <script src="scripts/search.js" defer></script> </body> </html>