UNPKG

matrix-js-sdk

Version:
1,086 lines (1,036 loc) 102 kB
"use strict"; var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault"); Object.defineProperty(exports, "__esModule", { value: true }); exports.MatrixCall = exports.CallType = exports.CallState = exports.CallParty = exports.CallEvent = exports.CallErrorCode = exports.CallError = exports.CallDirection = void 0; exports.createNewMatrixCall = createNewMatrixCall; exports.genCallID = genCallID; exports.setTracksEnabled = setTracksEnabled; exports.supportsMatrixCall = supportsMatrixCall; var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty")); var _uuid = require("uuid"); var _sdpTransform = require("sdp-transform"); var _logger = require("../logger"); var utils = _interopRequireWildcard(require("../utils")); var _event = require("../@types/event"); var _randomstring = require("../randomstring"); var _callEventTypes = require("./callEventTypes"); var _callFeed = require("./callFeed"); var _typedEventEmitter = require("../models/typed-event-emitter"); var _deviceinfo = require("../crypto/deviceinfo"); var _groupCall = require("./groupCall"); var _httpApi = require("../http-api"); function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); } function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; } function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; } function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; } var MediaType; (function (MediaType) { MediaType["AUDIO"] = "audio"; MediaType["VIDEO"] = "video"; })(MediaType || (MediaType = {})); var CodecName; // add more as needed // Used internally to specify modifications to codec parameters in SDP (function (CodecName) { CodecName["OPUS"] = "opus"; })(CodecName || (CodecName = {})); let CallState; exports.CallState = CallState; (function (CallState) { CallState["Fledgling"] = "fledgling"; CallState["InviteSent"] = "invite_sent"; CallState["WaitLocalMedia"] = "wait_local_media"; CallState["CreateOffer"] = "create_offer"; CallState["CreateAnswer"] = "create_answer"; CallState["Connecting"] = "connecting"; CallState["Connected"] = "connected"; CallState["Ringing"] = "ringing"; CallState["Ended"] = "ended"; })(CallState || (exports.CallState = CallState = {})); let CallType; exports.CallType = CallType; (function (CallType) { CallType["Voice"] = "voice"; CallType["Video"] = "video"; })(CallType || (exports.CallType = CallType = {})); let CallDirection; exports.CallDirection = CallDirection; (function (CallDirection) { CallDirection["Inbound"] = "inbound"; CallDirection["Outbound"] = "outbound"; })(CallDirection || (exports.CallDirection = CallDirection = {})); let CallParty; exports.CallParty = CallParty; (function (CallParty) { CallParty["Local"] = "local"; CallParty["Remote"] = "remote"; })(CallParty || (exports.CallParty = CallParty = {})); let CallEvent; exports.CallEvent = CallEvent; (function (CallEvent) { CallEvent["Hangup"] = "hangup"; CallEvent["State"] = "state"; CallEvent["Error"] = "error"; CallEvent["Replaced"] = "replaced"; CallEvent["LocalHoldUnhold"] = "local_hold_unhold"; CallEvent["RemoteHoldUnhold"] = "remote_hold_unhold"; CallEvent["HoldUnhold"] = "hold_unhold"; CallEvent["FeedsChanged"] = "feeds_changed"; CallEvent["AssertedIdentityChanged"] = "asserted_identity_changed"; CallEvent["LengthChanged"] = "length_changed"; CallEvent["DataChannel"] = "datachannel"; CallEvent["SendVoipEvent"] = "send_voip_event"; })(CallEvent || (exports.CallEvent = CallEvent = {})); let CallErrorCode; /** * The version field that we set in m.call.* events */ exports.CallErrorCode = CallErrorCode; (function (CallErrorCode) { CallErrorCode["UserHangup"] = "user_hangup"; CallErrorCode["LocalOfferFailed"] = "local_offer_failed"; CallErrorCode["NoUserMedia"] = "no_user_media"; CallErrorCode["UnknownDevices"] = "unknown_devices"; CallErrorCode["SendInvite"] = "send_invite"; CallErrorCode["CreateAnswer"] = "create_answer"; CallErrorCode["CreateOffer"] = "create_offer"; CallErrorCode["SendAnswer"] = "send_answer"; CallErrorCode["SetRemoteDescription"] = "set_remote_description"; CallErrorCode["SetLocalDescription"] = "set_local_description"; CallErrorCode["AnsweredElsewhere"] = "answered_elsewhere"; CallErrorCode["IceFailed"] = "ice_failed"; CallErrorCode["InviteTimeout"] = "invite_timeout"; CallErrorCode["Replaced"] = "replaced"; CallErrorCode["SignallingFailed"] = "signalling_timeout"; CallErrorCode["UserBusy"] = "user_busy"; CallErrorCode["Transfered"] = "transferred"; CallErrorCode["NewSession"] = "new_session"; })(CallErrorCode || (exports.CallErrorCode = CallErrorCode = {})); const VOIP_PROTO_VERSION = "1"; /** The fallback ICE server to use for STUN or TURN protocols. */ const FALLBACK_ICE_SERVER = "stun:turn.matrix.org"; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60 * 1000; // ms /** The time after which we increment callLength */ const CALL_LENGTH_INTERVAL = 1000; // ms /** The time after which we end the call, if ICE got disconnected */ const ICE_DISCONNECTED_TIMEOUT = 30 * 1000; // ms class CallError extends Error { constructor(code, msg, err) { // Still don't think there's any way to have proper nested errors super(msg + ": " + err); (0, _defineProperty2.default)(this, "code", void 0); this.code = code; } } exports.CallError = CallError; function genCallID() { return Date.now().toString() + (0, _randomstring.randomString)(16); } function getCodecParamMods(isPtt) { const mods = [{ mediaType: "audio", codec: "opus", enableDtx: true, maxAverageBitrate: isPtt ? 12000 : undefined }]; return mods; } // generates keys for the map of transceivers // kind is unfortunately a string rather than MediaType as this is the type of // track.kind function getTransceiverKey(purpose, kind) { return purpose + ":" + kind; } class MatrixCall extends _typedEventEmitter.TypedEventEmitter { // whether this call should have push-to-talk semantics // This should be set by the consumer on incoming & outgoing calls. // A queue for candidates waiting to go out. // We try to amalgamate candidates into a single candidate message where // possible // our transceivers for each purpose and type of media // The party ID of the other side: undefined if we haven't chosen a partner // yet, null if we have but they didn't send a party ID. // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold // This flag represents whether we want the other party to be on hold // the stats for the call at the point it ended. We can't get these after we // tear the call down, so we just grab a snapshot before we stop the call. // The typescript definitions have this type as 'any' :( // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick /** * Construct a new Matrix Call. * @param opts - Config options. */ constructor(opts) { var _opts$forceTURN; super(); (0, _defineProperty2.default)(this, "roomId", void 0); (0, _defineProperty2.default)(this, "callId", void 0); (0, _defineProperty2.default)(this, "invitee", void 0); (0, _defineProperty2.default)(this, "hangupParty", void 0); (0, _defineProperty2.default)(this, "hangupReason", void 0); (0, _defineProperty2.default)(this, "direction", void 0); (0, _defineProperty2.default)(this, "ourPartyId", void 0); (0, _defineProperty2.default)(this, "peerConn", void 0); (0, _defineProperty2.default)(this, "toDeviceSeq", 0); (0, _defineProperty2.default)(this, "isPtt", false); (0, _defineProperty2.default)(this, "_state", CallState.Fledgling); (0, _defineProperty2.default)(this, "client", void 0); (0, _defineProperty2.default)(this, "forceTURN", void 0); (0, _defineProperty2.default)(this, "turnServers", void 0); (0, _defineProperty2.default)(this, "candidateSendQueue", []); (0, _defineProperty2.default)(this, "candidateSendTries", 0); (0, _defineProperty2.default)(this, "candidatesEnded", false); (0, _defineProperty2.default)(this, "feeds", []); (0, _defineProperty2.default)(this, "transceivers", new Map()); (0, _defineProperty2.default)(this, "inviteOrAnswerSent", false); (0, _defineProperty2.default)(this, "waitForLocalAVStream", false); (0, _defineProperty2.default)(this, "successor", void 0); (0, _defineProperty2.default)(this, "opponentMember", void 0); (0, _defineProperty2.default)(this, "opponentVersion", void 0); (0, _defineProperty2.default)(this, "opponentPartyId", void 0); (0, _defineProperty2.default)(this, "opponentCaps", void 0); (0, _defineProperty2.default)(this, "iceDisconnectedTimeout", void 0); (0, _defineProperty2.default)(this, "inviteTimeout", void 0); (0, _defineProperty2.default)(this, "removeTrackListeners", new Map()); (0, _defineProperty2.default)(this, "remoteOnHold", false); (0, _defineProperty2.default)(this, "callStatsAtEnd", void 0); (0, _defineProperty2.default)(this, "makingOffer", false); (0, _defineProperty2.default)(this, "ignoreOffer", false); (0, _defineProperty2.default)(this, "responsePromiseChain", void 0); (0, _defineProperty2.default)(this, "remoteCandidateBuffer", new Map()); (0, _defineProperty2.default)(this, "remoteAssertedIdentity", void 0); (0, _defineProperty2.default)(this, "remoteSDPStreamMetadata", void 0); (0, _defineProperty2.default)(this, "callLengthInterval", void 0); (0, _defineProperty2.default)(this, "callStartTime", void 0); (0, _defineProperty2.default)(this, "opponentDeviceId", void 0); (0, _defineProperty2.default)(this, "opponentDeviceInfo", void 0); (0, _defineProperty2.default)(this, "opponentSessionId", void 0); (0, _defineProperty2.default)(this, "groupCallId", void 0); (0, _defineProperty2.default)(this, "gotLocalIceCandidate", event => { if (event.candidate) { if (this.candidatesEnded) { _logger.logger.warn("Got candidate after candidates have ended - ignoring!"); return; } _logger.logger.debug("Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + event.candidate.candidate); if (this.callHasEnded()) return; // As with the offer, note we need to make a copy of this object, not // pass the original: that broke in Chrome ~m43. if (event.candidate.candidate === "") { this.queueCandidate(null); } else { this.queueCandidate(event.candidate); } } }); (0, _defineProperty2.default)(this, "onIceGatheringStateChange", event => { var _this$peerConn; _logger.logger.debug(`Call ${this.callId} ice gathering state changed to ${this.peerConn.iceGatheringState}`); if (((_this$peerConn = this.peerConn) === null || _this$peerConn === void 0 ? void 0 : _this$peerConn.iceGatheringState) === "complete") { this.queueCandidate(null); } }); (0, _defineProperty2.default)(this, "getLocalOfferFailed", err => { _logger.logger.error(`Call ${this.callId} Failed to get local offer`, err); this.emit(CallEvent.Error, new CallError(CallErrorCode.LocalOfferFailed, "Failed to get local offer!", err)); this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false); }); (0, _defineProperty2.default)(this, "getUserMediaFailed", err => { if (this.successor) { this.successor.getUserMediaFailed(err); return; } _logger.logger.warn(`Failed to get user media - ending call ${this.callId}`, err); this.emit(CallEvent.Error, new CallError(CallErrorCode.NoUserMedia, "Couldn't start capturing media! Is your microphone set up and " + "does this app have permission?", err)); this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); }); (0, _defineProperty2.default)(this, "onIceConnectionStateChanged", () => { var _this$peerConn2, _this$peerConn$iceCon, _this$peerConn3, _this$peerConn4, _this$peerConn6; if (this.callHasEnded()) { return; // because ICE can still complete as we're ending the call } _logger.logger.debug("Call ID " + this.callId + ": ICE connection state changed to: " + ((_this$peerConn2 = this.peerConn) === null || _this$peerConn2 === void 0 ? void 0 : _this$peerConn2.iceConnectionState)); // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet if (["connected", "completed"].includes((_this$peerConn$iceCon = (_this$peerConn3 = this.peerConn) === null || _this$peerConn3 === void 0 ? void 0 : _this$peerConn3.iceConnectionState) !== null && _this$peerConn$iceCon !== void 0 ? _this$peerConn$iceCon : "")) { clearTimeout(this.iceDisconnectedTimeout); this.state = CallState.Connected; if (!this.callLengthInterval && !this.callStartTime) { this.callStartTime = Date.now(); this.callLengthInterval = setInterval(() => { this.emit(CallEvent.LengthChanged, Math.round((Date.now() - this.callStartTime) / 1000)); }, CALL_LENGTH_INTERVAL); } } else if (((_this$peerConn4 = this.peerConn) === null || _this$peerConn4 === void 0 ? void 0 : _this$peerConn4.iceConnectionState) == "failed") { var _this$peerConn5; // Firefox for Android does not yet have support for restartIce() // (the types say it's always defined though, so we have to cast // to prevent typescript from warning). if ((_this$peerConn5 = this.peerConn) !== null && _this$peerConn5 !== void 0 && _this$peerConn5.restartIce) { this.candidatesEnded = false; this.peerConn.restartIce(); } else { _logger.logger.info(`Hanging up call ${this.callId} (ICE failed and no ICE restart method)`); this.hangup(CallErrorCode.IceFailed, false); } } else if (((_this$peerConn6 = this.peerConn) === null || _this$peerConn6 === void 0 ? void 0 : _this$peerConn6.iceConnectionState) == "disconnected") { this.iceDisconnectedTimeout = setTimeout(() => { _logger.logger.info(`Hanging up call ${this.callId} (ICE disconnected for too long)`); this.hangup(CallErrorCode.IceFailed, false); }, ICE_DISCONNECTED_TIMEOUT); this.state = CallState.Connecting; } // In PTT mode, override feed status to muted when we lose connection to // the peer, since we don't want to block the line if they're not saying anything. // Experimenting in Chrome, this happens after 5 or 6 seconds, which is probably // fast enough. if (this.isPtt && ["failed", "disconnected"].includes(this.peerConn.iceConnectionState)) { for (const feed of this.getRemoteFeeds()) { feed.setAudioVideoMuted(true, true); } } }); (0, _defineProperty2.default)(this, "onSignallingStateChanged", () => { var _this$peerConn7; _logger.logger.debug(`call ${this.callId}: Signalling state changed to: ${(_this$peerConn7 = this.peerConn) === null || _this$peerConn7 === void 0 ? void 0 : _this$peerConn7.signalingState}`); }); (0, _defineProperty2.default)(this, "onTrack", ev => { if (ev.streams.length === 0) { _logger.logger.warn(`Call ${this.callId} Streamless ${ev.track.kind} found: ignoring.`); return; } const stream = ev.streams[0]; this.pushRemoteFeed(stream); if (!this.removeTrackListeners.has(stream)) { const onRemoveTrack = () => { if (stream.getTracks().length === 0) { _logger.logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); this.deleteFeedByStream(stream); stream.removeEventListener("removetrack", onRemoveTrack); this.removeTrackListeners.delete(stream); } }; stream.addEventListener("removetrack", onRemoveTrack); this.removeTrackListeners.set(stream, onRemoveTrack); } }); (0, _defineProperty2.default)(this, "onDataChannel", ev => { this.emit(CallEvent.DataChannel, ev.channel); }); (0, _defineProperty2.default)(this, "onNegotiationNeeded", async () => { _logger.logger.info(`Call ${this.callId} Negotiation is needed!`); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { _logger.logger.info(`Call ${this.callId} Opponent does not support renegotiation: ignoring negotiationneeded event`); return; } this.queueGotLocalOffer(); }); (0, _defineProperty2.default)(this, "onHangupReceived", msg => { _logger.logger.debug("Hangup received for call ID " + this.callId); // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen // a partner yet but we're treating the hangup as a reject as per VoIP v0) if (this.partyIdMatches(msg) || this.state === CallState.Ringing) { // default reason is user_hangup this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); } else { _logger.logger.info(`Call ${this.callId} Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); } }); (0, _defineProperty2.default)(this, "onRejectReceived", msg => { _logger.logger.debug("Reject received for call ID " + this.callId); // No need to check party_id for reject because if we'd received either // an answer or reject, we wouldn't be in state InviteSent const shouldTerminate = // reject events also end the call if it's ringing: it's another of // our devices rejecting the call. [CallState.InviteSent, CallState.Ringing].includes(this.state) || // also if we're in the init state and it's an inbound call, since // this means we just haven't entered the ringing state yet this.state === CallState.Fledgling && this.direction === CallDirection.Inbound; if (shouldTerminate) { this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); } else { _logger.logger.debug(`Call ${this.callId} is in state: ${this.state}: ignoring reject`); } }); (0, _defineProperty2.default)(this, "onAnsweredElsewhere", msg => { _logger.logger.debug("Call " + this.callId + " answered elsewhere"); this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); }); this.roomId = opts.roomId; this.invitee = opts.invitee; this.client = opts.client; if (!this.client.deviceId) throw new Error("Client must have a device ID to start calls"); this.forceTURN = (_opts$forceTURN = opts.forceTURN) !== null && _opts$forceTURN !== void 0 ? _opts$forceTURN : false; this.ourPartyId = this.client.deviceId; this.opponentDeviceId = opts.opponentDeviceId; this.opponentSessionId = opts.opponentSessionId; this.groupCallId = opts.groupCallId; // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { this.turnServers.push({ urls: [FALLBACK_ICE_SERVER] }); } for (const server of this.turnServers) { utils.checkObjectHasKeys(server, ["urls"]); } this.callId = genCallID(); } /** * Place a voice call to this room. * @throws If you have not specified a listener for 'error' events. */ async placeVoiceCall() { await this.placeCall(true, false); } /** * Place a video call to this room. * @throws If you have not specified a listener for 'error' events. */ async placeVideoCall() { await this.placeCall(true, true); } /** * Create a datachannel using this call's peer connection. * @param label - A human readable label for this datachannel * @param options - An object providing configuration options for the data channel. */ createDataChannel(label, options) { const dataChannel = this.peerConn.createDataChannel(label, options); this.emit(CallEvent.DataChannel, dataChannel); return dataChannel; } getOpponentMember() { return this.opponentMember; } getOpponentDeviceId() { return this.opponentDeviceId; } getOpponentSessionId() { return this.opponentSessionId; } opponentCanBeTransferred() { return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); } opponentSupportsDTMF() { return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]); } getRemoteAssertedIdentity() { return this.remoteAssertedIdentity; } get state() { return this._state; } set state(state) { const oldState = this._state; this._state = state; this.emit(CallEvent.State, state, oldState); } get type() { return this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack ? CallType.Video : CallType.Voice; } get hasLocalUserMediaVideoTrack() { var _this$localUsermediaS; return !!((_this$localUsermediaS = this.localUsermediaStream) !== null && _this$localUsermediaS !== void 0 && _this$localUsermediaS.getVideoTracks().length); } get hasRemoteUserMediaVideoTrack() { return this.getRemoteFeeds().some(feed => { var _feed$stream; return feed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Usermedia && ((_feed$stream = feed.stream) === null || _feed$stream === void 0 ? void 0 : _feed$stream.getVideoTracks().length); }); } get hasLocalUserMediaAudioTrack() { var _this$localUsermediaS2; return !!((_this$localUsermediaS2 = this.localUsermediaStream) !== null && _this$localUsermediaS2 !== void 0 && _this$localUsermediaS2.getAudioTracks().length); } get hasRemoteUserMediaAudioTrack() { return this.getRemoteFeeds().some(feed => { var _feed$stream2; return feed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Usermedia && !!((_feed$stream2 = feed.stream) !== null && _feed$stream2 !== void 0 && _feed$stream2.getAudioTracks().length); }); } get localUsermediaFeed() { return this.getLocalFeeds().find(feed => feed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Usermedia); } get localScreensharingFeed() { return this.getLocalFeeds().find(feed => feed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Screenshare); } get localUsermediaStream() { var _this$localUsermediaF; return (_this$localUsermediaF = this.localUsermediaFeed) === null || _this$localUsermediaF === void 0 ? void 0 : _this$localUsermediaF.stream; } get localScreensharingStream() { var _this$localScreenshar; return (_this$localScreenshar = this.localScreensharingFeed) === null || _this$localScreenshar === void 0 ? void 0 : _this$localScreenshar.stream; } get remoteUsermediaFeed() { return this.getRemoteFeeds().find(feed => feed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Usermedia); } get remoteScreensharingFeed() { return this.getRemoteFeeds().find(feed => feed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Screenshare); } get remoteUsermediaStream() { var _this$remoteUsermedia; return (_this$remoteUsermedia = this.remoteUsermediaFeed) === null || _this$remoteUsermedia === void 0 ? void 0 : _this$remoteUsermedia.stream; } get remoteScreensharingStream() { var _this$remoteScreensha; return (_this$remoteScreensha = this.remoteScreensharingFeed) === null || _this$remoteScreensha === void 0 ? void 0 : _this$remoteScreensha.stream; } getFeedByStreamId(streamId) { return this.getFeeds().find(feed => feed.stream.id === streamId); } /** * Returns an array of all CallFeeds * @returns CallFeeds */ getFeeds() { return this.feeds; } /** * Returns an array of all local CallFeeds * @returns local CallFeeds */ getLocalFeeds() { return this.feeds.filter(feed => feed.isLocal()); } /** * Returns an array of all remote CallFeeds * @returns remote CallFeeds */ getRemoteFeeds() { return this.feeds.filter(feed => !feed.isLocal()); } async initOpponentCrypto() { var _this$getOpponentMemb; if (!this.opponentDeviceId) return; if (!this.client.getUseE2eForGroupCall()) return; // It's possible to want E2EE and yet not have the means to manage E2EE // ourselves (for example if the client is a RoomWidgetClient) if (!this.client.isCryptoEnabled()) { // All we know is the device ID this.opponentDeviceInfo = new _deviceinfo.DeviceInfo(this.opponentDeviceId); return; } // if we've got to this point, we do want to init crypto, so throw if we can't if (!this.client.crypto) throw new Error("Crypto is not initialised."); const userId = this.invitee || ((_this$getOpponentMemb = this.getOpponentMember()) === null || _this$getOpponentMemb === void 0 ? void 0 : _this$getOpponentMemb.userId); if (!userId) throw new Error("Couldn't find opponent user ID to init crypto"); const deviceInfoMap = await this.client.crypto.deviceList.downloadKeys([userId], false); this.opponentDeviceInfo = deviceInfoMap[userId][this.opponentDeviceId]; if (this.opponentDeviceInfo === undefined) { throw new _groupCall.GroupCallUnknownDeviceError(userId); } } /** * Generates and returns localSDPStreamMetadata * @returns localSDPStreamMetadata */ getLocalSDPStreamMetadata(updateStreamIds = false) { const metadata = {}; for (const localFeed of this.getLocalFeeds()) { if (updateStreamIds) { localFeed.sdpMetadataStreamId = localFeed.stream.id; } metadata[localFeed.sdpMetadataStreamId] = { purpose: localFeed.purpose, audio_muted: localFeed.isAudioMuted(), video_muted: localFeed.isVideoMuted() }; } return metadata; } /** * Returns true if there are no incoming feeds, * otherwise returns false * @returns no incoming feeds */ noIncomingFeeds() { return !this.feeds.some(feed => !feed.isLocal()); } pushRemoteFeed(stream) { // Fallback to old behavior if the other side doesn't support SDPStreamMetadata if (!this.opponentSupportsSDPStreamMetadata()) { this.pushRemoteFeedWithoutMetadata(stream); return; } const userId = this.getOpponentMember().userId; const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted; const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; if (!purpose) { _logger.logger.warn(`Call ${this.callId} Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); return; } if (this.getFeedByStreamId(stream.id)) { _logger.logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); return; } this.feeds.push(new _callFeed.CallFeed({ client: this.client, call: this, roomId: this.roomId, userId, deviceId: this.getOpponentDeviceId(), stream, purpose, audioMuted, videoMuted })); this.emit(CallEvent.FeedsChanged, this.feeds); _logger.logger.info(`Call ${this.callId} pushed remote stream (id="${stream.id}", ` + `active="${stream.active}", purpose=${purpose})`); } /** * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata */ pushRemoteFeedWithoutMetadata(stream) { var _this$feeds$find; const userId = this.getOpponentMember().userId; // We can guess the purpose here since the other client can only send one stream const purpose = _callEventTypes.SDPStreamMetadataPurpose.Usermedia; const oldRemoteStream = (_this$feeds$find = this.feeds.find(feed => !feed.isLocal())) === null || _this$feeds$find === void 0 ? void 0 : _this$feeds$find.stream; // Note that we check by ID and always set the remote stream: Chrome appears // to make new stream objects when transceiver directionality is changed and the 'active' // status of streams change - Dave // If we already have a stream, check this stream has the same id if (oldRemoteStream && stream.id !== oldRemoteStream.id) { _logger.logger.warn(`Call ${this.callId} Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); return; } if (this.getFeedByStreamId(stream.id)) { _logger.logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); return; } this.feeds.push(new _callFeed.CallFeed({ client: this.client, call: this, roomId: this.roomId, audioMuted: false, videoMuted: false, userId, deviceId: this.getOpponentDeviceId(), stream, purpose })); this.emit(CallEvent.FeedsChanged, this.feeds); _logger.logger.info(`Call ${this.callId} pushed remote stream (id="${stream.id}", active="${stream.active}")`); } pushNewLocalFeed(stream, purpose, addToPeerConnection = true) { const userId = this.client.getUserId(); // Tracks don't always start off enabled, eg. chrome will give a disabled // audio track if you ask for user media audio and already had one that // you'd set to disabled (presumably because it clones them internally). setTracksEnabled(stream.getAudioTracks(), true); setTracksEnabled(stream.getVideoTracks(), true); if (this.getFeedByStreamId(stream.id)) { _logger.logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); return; } this.pushLocalFeed(new _callFeed.CallFeed({ client: this.client, roomId: this.roomId, audioMuted: false, videoMuted: false, userId, deviceId: this.getOpponentDeviceId(), stream, purpose }), addToPeerConnection); } /** * Pushes supplied feed to the call * @param callFeed - to push * @param addToPeerConnection - whether to add the tracks to the peer connection */ pushLocalFeed(callFeed, addToPeerConnection = true) { if (this.feeds.some(feed => callFeed.stream.id === feed.stream.id)) { _logger.logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); return; } this.feeds.push(callFeed); if (addToPeerConnection) { for (const track of callFeed.stream.getTracks()) { _logger.logger.info(`Call ${this.callId} ` + `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + `streamId="${callFeed.stream.id}", ` + `streamPurpose="${callFeed.purpose}", ` + `enabled=${track.enabled}` + `) to peer connection`); const tKey = getTransceiverKey(callFeed.purpose, track.kind); if (this.transceivers.has(tKey)) { // we already have a sender, so we re-use it. We try to re-use transceivers as much // as possible because they can't be removed once added, so otherwise they just // accumulate which makes the SDP very large very quickly: in fact it only takes // about 6 video tracks to exceed the maximum size of an Olm-encrypted // Matrix event. const transceiver = this.transceivers.get(tKey); // this is what would allow us to use addTransceiver(), but it's not available // on Firefox yet. We call it anyway if we have it. if (transceiver.sender.setStreams) transceiver.sender.setStreams(callFeed.stream); transceiver.sender.replaceTrack(track); // set the direction to indicate we're going to start sending again // (this will trigger the re-negotiation) transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv"; } else { // create a new one. We need to use addTrack rather addTransceiver for this because firefox // doesn't yet implement RTCRTPSender.setStreams() // (https://bugzilla.mozilla.org/show_bug.cgi?id=1510802) so we'd have no way to group the // two tracks together into a stream. const newSender = this.peerConn.addTrack(track, callFeed.stream); // now go & fish for the new transceiver const newTransciever = this.peerConn.getTransceivers().find(t => t.sender === newSender); if (newTransciever) { this.transceivers.set(tKey, newTransciever); } else { _logger.logger.warn("Didn't find a matching transceiver after adding track!"); } } } } _logger.logger.info(`Call ${this.callId} ` + `Pushed local stream ` + `(id="${callFeed.stream.id}", ` + `active="${callFeed.stream.active}", ` + `purpose="${callFeed.purpose}")`); this.emit(CallEvent.FeedsChanged, this.feeds); } /** * Removes local call feed from the call and its tracks from the peer * connection * @param callFeed - to remove */ removeLocalFeed(callFeed) { const audioTransceiverKey = getTransceiverKey(callFeed.purpose, "audio"); const videoTransceiverKey = getTransceiverKey(callFeed.purpose, "video"); for (const transceiverKey of [audioTransceiverKey, videoTransceiverKey]) { // this is slightly mixing the track and transceiver API but is basically just shorthand. // There is no way to actually remove a transceiver, so this just sets it to inactive // (or recvonly) and replaces the source with nothing. if (this.transceivers.has(transceiverKey)) { const transceiver = this.transceivers.get(transceiverKey); if (transceiver.sender) this.peerConn.removeTrack(transceiver.sender); } } if (callFeed.purpose === _callEventTypes.SDPStreamMetadataPurpose.Screenshare) { this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); } this.deleteFeed(callFeed); } deleteAllFeeds() { for (const feed of this.feeds) { if (!feed.isLocal() || !this.groupCallId) { feed.dispose(); } } this.feeds = []; this.emit(CallEvent.FeedsChanged, this.feeds); } deleteFeedByStream(stream) { const feed = this.getFeedByStreamId(stream.id); if (!feed) { _logger.logger.warn(`Call ${this.callId} Didn't find the feed with stream id ${stream.id} to delete`); return; } this.deleteFeed(feed); } deleteFeed(feed) { feed.dispose(); this.feeds.splice(this.feeds.indexOf(feed), 1); this.emit(CallEvent.FeedsChanged, this.feeds); } // The typescript definitions have this type as 'any' :( async getCurrentCallStats() { if (this.callHasEnded()) { return this.callStatsAtEnd; } return this.collectCallStats(); } async collectCallStats() { // This happens when the call fails before it starts. // For example when we fail to get capture sources if (!this.peerConn) return; const statsReport = await this.peerConn.getStats(); const stats = []; statsReport.forEach(item => { stats.push(item); }); return stats; } /** * Configure this call from an invite event. Used by MatrixClient. * @param event - The m.call.invite event */ async initWithInvite(event) { var _this$feeds$find2; const invite = event.getContent(); this.direction = CallDirection.Inbound; // make sure we have valid turn creds. Unless something's gone wrong, it should // poll and keep the credentials valid so this should be instant. const haveTurnCreds = await this.client.checkTurnServers(); if (!haveTurnCreds) { _logger.logger.warn(`Call ${this.callId} Failed to get TURN credentials! Proceeding with call anyway...`); } const sdpStreamMetadata = invite[_callEventTypes.SDPStreamMetadataKey]; if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { _logger.logger.debug(`Call ${this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } this.peerConn = this.createPeerConnection(); // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if // we haven't set the party ID, we'll ignore them. this.chooseOpponent(event); await this.initOpponentCrypto(); try { await this.peerConn.setRemoteDescription(invite.offer); await this.addBufferedIceCandidates(); } catch (e) { _logger.logger.debug(`Call ${this.callId} failed to set remote description`, e); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } const remoteStream = (_this$feeds$find2 = this.feeds.find(feed => !feed.isLocal())) === null || _this$feeds$find2 === void 0 ? void 0 : _this$feeds$find2.stream; // According to previous comments in this file, firefox at some point did not // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (!remoteStream || remoteStream.getTracks().length === 0) { _logger.logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } this.state = CallState.Ringing; if (event.getLocalAge()) { // Time out the call if it's ringing for too long const ringingTimer = setTimeout(() => { if (this.state == CallState.Ringing) { _logger.logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.state = CallState.Ended; this.stopAllMedia(); if (this.peerConn.signalingState != "closed") { this.peerConn.close(); } this.emit(CallEvent.Hangup, this); } }, invite.lifetime - event.getLocalAge()); const onState = state => { if (state !== CallState.Ringing) { clearTimeout(ringingTimer); this.off(CallEvent.State, onState); } }; this.on(CallEvent.State, onState); } } /** * Configure this call from a hangup or reject event. Used by MatrixClient. * @param event - The m.call.hangup event */ initWithHangup(event) { // perverse as it may seem, sometimes we want to instantiate a call with a // hangup message (because when getting the state of the room on load, events // come in reverse order and we want to remember that a call has been hung up) this.state = CallState.Ended; } shouldAnswerWithMediaType(wantedValue, valueOfTheOtherSide, type) { if (wantedValue && !valueOfTheOtherSide) { // TODO: Figure out how to do this _logger.logger.warn(`Call ${this.callId} Unable to answer with ${type} because the other side isn't sending it either.`); return false; } else if (!utils.isNullOrUndefined(wantedValue) && wantedValue !== valueOfTheOtherSide && !this.opponentSupportsSDPStreamMetadata()) { _logger.logger.warn(`Call ${this.callId} Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. Answering with ${type}=${valueOfTheOtherSide}.`); return valueOfTheOtherSide; } return wantedValue !== null && wantedValue !== void 0 ? wantedValue : valueOfTheOtherSide; } /** * Answer a call. */ async answer(audio, video) { if (this.inviteOrAnswerSent) return; // TODO: Figure out how to do this if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); if (!this.localUsermediaStream && !this.waitForLocalAVStream) { const prevState = this.state; const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio"); const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"); this.state = CallState.WaitLocalMedia; this.waitForLocalAVStream = true; try { var _this$client$getDevic; const stream = await this.client.getMediaHandler().getUserMediaStream(answerWithAudio, answerWithVideo); this.waitForLocalAVStream = false; const usermediaFeed = new _callFeed.CallFeed({ client: this.client, roomId: this.roomId, userId: this.client.getUserId(), deviceId: (_this$client$getDevic = this.client.getDeviceId()) !== null && _this$client$getDevic !== void 0 ? _this$client$getDevic : undefined, stream, purpose: _callEventTypes.SDPStreamMetadataPurpose.Usermedia, audioMuted: false, videoMuted: false }); const feeds = [usermediaFeed]; if (this.localScreensharingFeed) { feeds.push(this.localScreensharingFeed); } this.answerWithCallFeeds(feeds); } catch (e) { if (answerWithVideo) { // Try to answer without video _logger.logger.warn(`Call ${this.callId} Failed to getUserMedia(), trying to getUserMedia() without video`); this.state = prevState; this.waitForLocalAVStream = false; await this.answer(answerWithAudio, false); } else { this.getUserMediaFailed(e); return; } } } else if (this.waitForLocalAVStream) { this.state = CallState.WaitLocalMedia; } } answerWithCallFeeds(callFeeds) { if (this.inviteOrAnswerSent) return; this.queueGotCallFeedsForAnswer(callFeeds); } /** * Replace this call with a new call, e.g. for glare resolution. Used by * MatrixClient. * @param newCall - The new call. */ replacedBy(newCall) { _logger.logger.debug(`Call ${this.callId} replaced by ${newCall.callId}`); if (this.state === CallState.WaitLocalMedia) { _logger.logger.debug(`Call ${this.callId} telling new call ${newCall.callId} to wait for local media`); newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { if (newCall.direction === CallDirection.Outbound) { newCall.queueGotCallFeedsForAnswer([]); } else { _logger.logger.debug(`Call ${this.callId} handing local stream to new call ${newCall.callId}`); newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); } } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); this.hangup(CallErrorCode.Replaced, true); } /** * Hangup a call. * @param reason - The reason why the call is being hung up. * @param suppressEvent - True to suppress emitting an event. */ hangup(reason, suppressEvent) { if (this.callHasEnded()) return; _logger.logger.debug(`Ending call ${this.callId} with reason ${reason}`); this.terminate(CallParty.Local, reason, !suppressEvent); // We don't want to send hangup here if we didn't even get to sending an invite if ([CallState.Fledgling, CallState.WaitLocalMedia].includes(this.state)) return; const content = {}; // Don't send UserHangup reason to older clients if (this.opponentVersion && this.opponentVersion !== 0 || reason !== CallErrorCode.UserHangup) { content["reason"] = reason; } this.sendVoipEvent(_event.EventType.CallHangup, content); } /** * Reject a call * This used to be done by calling hangup, but is a separate method and protocol * event as of MSC2746. */ reject() { if (this.state !== CallState.Ringing) { throw Error("Call must be in 'ringing' state to reject!"); } if (this.opponentVersion === 0) { _logger.logger.info(`Call ${this.callId} Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`); this.hangup(CallErrorCode.UserHangup, true); return; } _logger.logger.debug("Rejecting call: " + this.callId); this.terminate(CallParty.Local, CallErrorCode.UserHangup, true); this.sendVoipEvent(_event.EventType.CallReject, {}); } /** * Adds an audio and/or video track - upgrades the call * @param audio - should add an audio track * @param video - should add an video track */ async upgradeCall(audio, video) { // We don't do call downgrades if (!audio && !video) return; if (!this.opponentSupportsSDPStreamMetadata()) return; try { _logger.logger.debug(`Upgrading call ${this.callId}: audio?=${audio} video?=${video}`); const getAudio = audio || this.hasLocalUserMediaAudioTrack; const getVideo = video || this.hasLocalUserMediaVideoTrack; // updateLocalUsermediaStream() will take the tracks, use them as // replacement and throw the stream away, so it isn't reusable const stream = await this.client.getMediaHandler().getUserMediaStream(getAudio, getVideo, false); await this.updateLocalUsermediaStream(stream, audio, video); } catch (error) { _logger.logger.error(`Call ${this.callId} Failed to upgrade the call`, error); this.emit(CallEvent.Error, new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error)); } } /** * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false * @returns can screenshare */ opponentSupportsSDPStreamMetadata() { return Boolean(this.remoteSDPStreamMetadata); } /** * If there is a screensharing stream returns true, otherwise returns false * @returns is screensharing */ isScreensharing() { return Boolean(this.localScreensharingStream); } /** * Starts/stops screensharing * @param enabled - the desired screensharing state * @param desktopCapturerSourceId - optional id of the desktop capturer source to use * @returns new screensharing state */ async setScreensharingEnabled(enabled, opts) { // Skip if there is nothing to do if (enabled && this.isScreensharing()) { _logger.logger.warn(`Call ${this.callId} There is already a screensharing stream - there is nothing to do!`); return true; } else if (!enabled && !this.isScreensharing()) { _logger.logger.warn(`Call ${this.callId} There already isn't a screensharing stream - there is nothing to do!`); return false; } // Fallback to replaceTrack() if (!this.opponentSupportsSDPStreamMetadata()) { return this.setScreensharingEnabledWithoutMetadataSupport(enabled, opts); } _logger.logger.debug(`Call ${this.callId} set screensharing enabled? ${enabled}`); if (enabled) { try { const stream = await this.client.getMediaHandler().getScreensharingStream(opts); if (!stream) return false; this.pushNewLocalFeed(stream, _callEventTypes.SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { _logger.logger.error(`Call ${this.callId} Failed to get screen-sharing stream:`, err); return false; } } else { const audioTransceiver = this.transceivers.get(getTransceiverKey(_callEventTypes.SDPStreamMetadataPurpose.Screenshare, "audio")); const videoTransceiver = this.transceivers.get(getTransceiverKey(_callEventTypes.SDPStreamMetadataPurpose.Screenshare, "video")); for (const transceiver of [audioTransceiver, videoTransceiver]) { // this is slightly mixing the track and transceiver API but is basically just shorthand // for removing the sender. if (transceiver && transceiver.sender) this.peerConn.removeTrack(transceiver.sender); } this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); this.deleteFeedByStream(this.localScreensharingStream); return false; } } /** * Starts/stops screensharing *