matrix-js-sdk/src/webrtc/call.ts

Matrix Client-Server SDK for Javascript

/*
Copyright 2015, 2016 OpenMarket Ltd
Copyright 2017 New Vector Ltd
Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
Copyright 2021 - 2022 Šimon Brandner <simon.bra.ag@gmail.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/**
 * This is an internal module. See {@link createNewMatrixCall} for the public API.
 * @module webrtc/call
 */

import { logger } from '../logger';
import * as utils from '../utils';
import { MatrixEvent } from '../models/event';
import { EventType } from '../@types/event';
import { RoomMember } from '../models/room-member';
import { randomString } from '../randomstring';
import {
    MCallReplacesEvent,
    MCallAnswer,
    MCallInviteNegotiate,
    CallCapabilities,
    SDPStreamMetadataPurpose,
    SDPStreamMetadata,
    SDPStreamMetadataKey,
    MCallSDPStreamMetadataChanged,
    MCallSelectAnswer,
    MCAllAssertedIdentity,
    MCallCandidates,
    MCallBase,
    MCallHangupReject,
} from './callEventTypes';
import { CallFeed } from './callFeed';
import { MatrixClient } from "../client";
import { ISendEventResponse } from "../@types/requests";
import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter";

// events: hangup, error(err), replaced(call), state(state, oldState)

/**
 * Fires whenever an error occurs when call.js encounters an issue with setting up the call.
 * <p>
 * The error given will have a code equal to either `MatrixCall.ERR_LOCAL_OFFER_FAILED` or
 * `MatrixCall.ERR_NO_USER_MEDIA`. `ERR_LOCAL_OFFER_FAILED` is emitted when the local client
 * fails to create an offer. `ERR_NO_USER_MEDIA` is emitted when the user has denied access
 * to their audio/video hardware.
 *
 * @event module:webrtc/call~MatrixCall#"error"
 * @param {Error} err The error raised by MatrixCall.
 * @example
 * matrixCall.on("error", function(err){
 *   console.error(err.code, err);
 * });
 */

interface CallOpts {
    roomId?: string;
    client?: any; // Fix when client is TSified
    forceTURN?: boolean;
    turnServers?: Array<TurnServer>;
}

interface TurnServer {
    urls: Array<string>;
    username?: string;
    password?: string;
    ttl?: number;
}

interface AssertedIdentity {
    id: string;
    displayName: string;
}

export enum CallState {
    Fledgling = 'fledgling',
    InviteSent = 'invite_sent',
    WaitLocalMedia = 'wait_local_media',
    CreateOffer = 'create_offer',
    CreateAnswer = 'create_answer',
    Connecting = 'connecting',
    Connected = 'connected',
    Ringing = 'ringing',
    Ended = 'ended',
}

export enum CallType {
    Voice = 'voice',
    Video = 'video',
}

export enum CallDirection {
    Inbound = 'inbound',
    Outbound = 'outbound',
}

export enum CallParty {
    Local = 'local',
    Remote = 'remote',
}

export enum CallEvent {
    Hangup = 'hangup',
    State = 'state',
    Error = 'error',
    Replaced = 'replaced',

    // The value of isLocalOnHold() has changed
    LocalHoldUnhold = 'local_hold_unhold',
    // The value of isRemoteOnHold() has changed
    RemoteHoldUnhold = 'remote_hold_unhold',
    // backwards compat alias for LocalHoldUnhold: remove in a major version bump
    HoldUnhold = 'hold_unhold',
    // Feeds have changed
    FeedsChanged = 'feeds_changed',

    AssertedIdentityChanged = 'asserted_identity_changed',

    LengthChanged = 'length_changed',

    DataChannel = 'datachannel',
}

export enum CallErrorCode {
    /** The user chose to end the call */
    UserHangup = 'user_hangup',

    /** An error code when the local client failed to create an offer. */
    LocalOfferFailed = 'local_offer_failed',
    /**
     * An error code when there is no local mic/camera to use. This may be because
     * the hardware isn't plugged in, or the user has explicitly denied access.
     */
    NoUserMedia = 'no_user_media',

    /**
     * Error code used when a call event failed to send
     * because unknown devices were present in the room
     */
    UnknownDevices = 'unknown_devices',

    /**
     * Error code used when we fail to send the invite
     * for some reason other than there being unknown devices
     */
    SendInvite = 'send_invite',

    /**
     * An answer could not be created
     */
    CreateAnswer = 'create_answer',

    /**
     * Error code used when we fail to send the answer
     * for some reason other than there being unknown devices
     */
    SendAnswer = 'send_answer',

    /**
     * The session description from the other side could not be set
     */
    SetRemoteDescription = 'set_remote_description',

    /**
     * The session description from this side could not be set
     */
    SetLocalDescription = 'set_local_description',

    /**
     * A different device answered the call
     */
    AnsweredElsewhere = 'answered_elsewhere',

    /**
     * No media connection could be established to the other party
     */
    IceFailed = 'ice_failed',

    /**
     * The invite timed out whilst waiting for an answer
     */
    InviteTimeout = 'invite_timeout',

    /**
     * The call was replaced by another call
     */
    Replaced = 'replaced',

    /**
     * Signalling for the call could not be sent (other than the initial invite)
     */
    SignallingFailed = 'signalling_timeout',

    /**
     * The remote party is busy
     */
    UserBusy = 'user_busy',

    /**
     * We transferred the call off to somewhere else
     */
    Transfered = 'transferred',
}

/**
 * The version field that we set in m.call.* events
 */
const VOIP_PROTO_VERSION = "1";

/** The fallback ICE server to use for STUN or TURN protocols. */
const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org';

/** The length of time a call can be ringing for. */
const CALL_TIMEOUT_MS = 60000;

export class CallError extends Error {
    code: string;

    constructor(code: CallErrorCode, msg: string, err: Error) {
        // Still don't think there's any way to have proper nested errors
        super(msg + ": " + err);

        this.code = code;
    }
}

function genCallID(): string {
    return Date.now().toString() + randomString(16);
}

export type CallEventHandlerMap = {
    [CallEvent.DataChannel]: (channel: RTCDataChannel) => void;
    [CallEvent.FeedsChanged]: (feeds: CallFeed[]) => void;
    [CallEvent.Replaced]: (newCall: MatrixCall) => void;
    [CallEvent.Error]: (error: CallError) => void;
    [CallEvent.RemoteHoldUnhold]: (onHold: boolean) => void;
    [CallEvent.LocalHoldUnhold]: (onHold: boolean) => void;
    [CallEvent.LengthChanged]: (length: number) => void;
    [CallEvent.State]: (state: CallState, oldState?: CallState) => void;
    [CallEvent.Hangup]: () => void;
    [CallEvent.AssertedIdentityChanged]: () => void;
    /* @deprecated */
    [CallEvent.HoldUnhold]: (onHold: boolean) => void;
};

/**
 * Construct a new Matrix Call.
 * @constructor
 * @param {Object} opts Config options.
 * @param {string} opts.roomId The room ID for this call.
 * @param {Object} opts.webRtc The WebRTC globals from the browser.
 * @param {boolean} opts.forceTURN whether relay through TURN should be forced.
 * @param {Object} opts.URL The URL global.
 * @param {Array<Object>} opts.turnServers Optional. A list of TURN servers.
 * @param {MatrixClient} opts.client The Matrix Client instance to send events to.
 */
export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap> {
    public roomId: string;
    public callId: string;
    public state = CallState.Fledgling;
    public hangupParty: CallParty;
    public hangupReason: string;
    public direction: CallDirection;
    public ourPartyId: string;

    private client: MatrixClient;
    private forceTURN: boolean;
    private turnServers: Array<TurnServer>;
    // A queue for candidates waiting to go out.
    // We try to amalgamate candidates into a single candidate message where
    // possible
    private candidateSendQueue: Array<RTCIceCandidate> = [];
    private candidateSendTries = 0;
    private sentEndOfCandidates = false;
    private peerConn: RTCPeerConnection;
    private feeds: Array<CallFeed> = [];
    private usermediaSenders: Array<RTCRtpSender> = [];
    private screensharingSenders: Array<RTCRtpSender> = [];
    private inviteOrAnswerSent = false;
    private waitForLocalAVStream: boolean;
    private successor: MatrixCall;
    private opponentMember: RoomMember;
    private opponentVersion: number | string;
    // The party ID of the other side: undefined if we haven't chosen a partner
    // yet, null if we have but they didn't send a party ID.
    private opponentPartyId: string;
    private opponentCaps: CallCapabilities;
    private inviteTimeout: ReturnType<typeof setTimeout>;

    // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold
    // This flag represents whether we want the other party to be on hold
    private remoteOnHold = false;

    // the stats for the call at the point it ended. We can't get these after we
    // tear the call down, so we just grab a snapshot before we stop the call.
    // The typescript definitions have this type as 'any' :(
    private callStatsAtEnd: any[];

    // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example
    private makingOffer = false;
    private ignoreOffer: boolean;

    // If candidates arrive before we've picked an opponent (which, in particular,
    // will happen if the opponent sends candidates eagerly before the user answers
    // the call) we buffer them up here so we can then add the ones from the party we pick
    private remoteCandidateBuffer = new Map<string, RTCIceCandidate[]>();

    private remoteAssertedIdentity: AssertedIdentity;

    private remoteSDPStreamMetadata: SDPStreamMetadata;

    private callLengthInterval: ReturnType<typeof setInterval>;
    private callLength = 0;

    constructor(opts: CallOpts) {
        super();
        this.roomId = opts.roomId;
        this.client = opts.client;
        this.forceTURN = opts.forceTURN;
        this.ourPartyId = this.client.deviceId;
        // Array of Objects with urls, username, credential keys
        this.turnServers = opts.turnServers || [];
        if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) {
            this.turnServers.push({
                urls: [FALLBACK_ICE_SERVER],
            });
        }
        for (const server of this.turnServers) {
            utils.checkObjectHasKeys(server, ["urls"]);
        }
        this.callId = genCallID();
    }

    /**
     * Place a voice call to this room.
     * @throws If you have not specified a listener for 'error' events.
     */
    public async placeVoiceCall(): Promise<void> {
        await this.placeCall(true, false);
    }

    /**
     * Place a video call to this room.
     * @throws If you have not specified a listener for 'error' events.
     */
    public async placeVideoCall(): Promise<void> {
        await this.placeCall(true, true);
    }

    /**
     * Create a datachannel using this call's peer connection.
     * @param label A human readable label for this datachannel
     * @param options An object providing configuration options for the data channel.
     */
    public createDataChannel(label: string, options: RTCDataChannelInit) {
        const dataChannel = this.peerConn.createDataChannel(label, options);
        this.emit(CallEvent.DataChannel, dataChannel);
        logger.debug("created data channel");
        return dataChannel;
    }

    public getOpponentMember(): RoomMember {
        return this.opponentMember;
    }

    public opponentCanBeTransferred(): boolean {
        return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]);
    }

    public opponentSupportsDTMF(): boolean {
        return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]);
    }

    public getRemoteAssertedIdentity(): AssertedIdentity {
        return this.remoteAssertedIdentity;
    }

    public get type(): CallType {
        return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack)
            ? CallType.Video
            : CallType.Voice;
    }

    public get hasLocalUserMediaVideoTrack(): boolean {
        return this.localUsermediaStream?.getVideoTracks().length > 0;
    }

    public get hasRemoteUserMediaVideoTrack(): boolean {
        return this.getRemoteFeeds().some((feed) => {
            return (
                feed.purpose === SDPStreamMetadataPurpose.Usermedia &&
                feed.stream.getVideoTracks().length > 0
            );
        });
    }

    public get hasLocalUserMediaAudioTrack(): boolean {
        return this.localUsermediaStream?.getAudioTracks().length > 0;
    }

    public get hasRemoteUserMediaAudioTrack(): boolean {
        return this.getRemoteFeeds().some((feed) => {
            return (
                feed.purpose === SDPStreamMetadataPurpose.Usermedia &&
                feed.stream.getAudioTracks().length > 0
            );
        });
    }

    public get localUsermediaFeed(): CallFeed {
        return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
    }

    public get localScreensharingFeed(): CallFeed {
        return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
    }

    public get localUsermediaStream(): MediaStream {
        return this.localUsermediaFeed?.stream;
    }

    public get localScreensharingStream(): MediaStream {
        return this.localScreensharingFeed?.stream;
    }

    public get remoteUsermediaFeed(): CallFeed {
        return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
    }

    public get remoteScreensharingFeed(): CallFeed {
        return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
    }

    public get remoteUsermediaStream(): MediaStream {
        return this.remoteUsermediaFeed?.stream;
    }

    public get remoteScreensharingStream(): MediaStream {
        return this.remoteScreensharingFeed?.stream;
    }

    private getFeedByStreamId(streamId: string): CallFeed {
        return this.getFeeds().find((feed) => feed.stream.id === streamId);
    }

    /**
     * Returns an array of all CallFeeds
     * @returns {Array<CallFeed>} CallFeeds
     */
    public getFeeds(): Array<CallFeed> {
        return this.feeds;
    }

    /**
     * Returns an array of all local CallFeeds
     * @returns {Array<CallFeed>} local CallFeeds
     */
    public getLocalFeeds(): Array<CallFeed> {
        return this.feeds.filter((feed) => feed.isLocal());
    }

    /**
     * Returns an array of all remote CallFeeds
     * @returns {Array<CallFeed>} remote CallFeeds
     */
    public getRemoteFeeds(): Array<CallFeed> {
        return this.feeds.filter((feed) => !feed.isLocal());
    }

    /**
     * Generates and returns localSDPStreamMetadata
     * @returns {SDPStreamMetadata} localSDPStreamMetadata
     */
    private getLocalSDPStreamMetadata(): SDPStreamMetadata {
        const metadata: SDPStreamMetadata = {};
        for (const localFeed of this.getLocalFeeds()) {
            metadata[localFeed.stream.id] = {
                purpose: localFeed.purpose,
                audio_muted: localFeed.isAudioMuted(),
                video_muted: localFeed.isVideoMuted(),
            };
        }
        logger.debug("Got local SDPStreamMetadata", metadata);
        return metadata;
    }

    /**
     * Returns true if there are no incoming feeds,
     * otherwise returns false
     * @returns {boolean} no incoming feeds
     */
    public noIncomingFeeds(): boolean {
        return !this.feeds.some((feed) => !feed.isLocal());
    }

    private pushRemoteFeed(stream: MediaStream): void {
        // Fallback to old behavior if the other side doesn't support SDPStreamMetadata
        if (!this.opponentSupportsSDPStreamMetadata()) {
            this.pushRemoteFeedWithoutMetadata(stream);
            return;
        }

        const userId = this.getOpponentMember().userId;
        const purpose = this.remoteSDPStreamMetadata[stream.id].purpose;
        const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted;
        const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted;

        if (!purpose) {
            logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`);
            return;
        }

        if (this.getFeedByStreamId(stream.id)) {
            logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`);
            return;
        }

        this.feeds.push(new CallFeed({
            client: this.client,
            roomId: this.roomId,
            userId,
            stream,
            purpose,
            audioMuted,
            videoMuted,
        }));
        this.emit(CallEvent.FeedsChanged, this.feeds);

        logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`);
    }

    /**
     * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata
     */
    private pushRemoteFeedWithoutMetadata(stream: MediaStream): void {
        const userId = this.getOpponentMember().userId;
        // We can guess the purpose here since the other client can only send one stream
        const purpose = SDPStreamMetadataPurpose.Usermedia;
        const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream;

        // Note that we check by ID and always set the remote stream: Chrome appears
        // to make new stream objects when transceiver directionality is changed and the 'active'
        // status of streams change - Dave
        // If we already have a stream, check this stream has the same id
        if (oldRemoteStream && stream.id !== oldRemoteStream.id) {
            logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`);
            return;
        }

        if (this.getFeedByStreamId(stream.id)) {
            logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`);
            return;
        }

        this.feeds.push(new CallFeed({
            client: this.client,
            roomId: this.roomId,
            audioMuted: false,
            videoMuted: false,
            userId,
            stream,
            purpose,
        }));
        this.emit(CallEvent.FeedsChanged, this.feeds);

        logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`);
    }

    private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void {
        const userId = this.client.getUserId();

        // Tracks don't always start off enabled, eg. chrome will give a disabled
        // audio track if you ask for user media audio and already had one that
        // you'd set to disabled (presumably because it clones them internally).
        setTracksEnabled(stream.getAudioTracks(), true);
        setTracksEnabled(stream.getVideoTracks(), true);

        if (this.getFeedByStreamId(stream.id)) {
            logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`);
            return;
        }

        this.pushLocalFeed(
            new CallFeed({
                client: this.client,
                roomId: this.roomId,
                audioMuted: false,
                videoMuted: false,
                userId,
                stream,
                purpose,
            }),
            addToPeerConnection,
        );
    }

    /**
     * Pushes supplied feed to the call
     * @param {CallFeed} callFeed to push
     * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection
     */
    public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void {
        this.feeds.push(callFeed);

        if (addToPeerConnection) {
            const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ?
                this.usermediaSenders : this.screensharingSenders;
            // Empty the array
            senderArray.splice(0, senderArray.length);

            for (const track of callFeed.stream.getTracks()) {
                logger.info(
                    `Adding track (` +
                    `id="${track.id}", ` +
                    `kind="${track.kind}", ` +
                    `streamId="${callFeed.stream.id}", ` +
                    `streamPurpose="${callFeed.purpose}", ` +
                    `enabled=${track.enabled}` +
                    `) to peer connection`,
                );
                senderArray.push(this.peerConn.addTrack(track, callFeed.stream));
            }
        }

        logger.info(
            `Pushed local stream ` +
            `(id="${callFeed.stream.id}", ` +
            `active="${callFeed.stream.active}", ` +
            `purpose="${callFeed.purpose}")`,
        );

        this.emit(CallEvent.FeedsChanged, this.feeds);
    }

    /**
     * Removes local call feed from the call and its tracks from the peer
     * connection
     * @param callFeed to remove
     */
    public removeLocalFeed(callFeed: CallFeed): void {
        const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia
            ? this.usermediaSenders
            : this.screensharingSenders;

        for (const sender of senderArray) {
            this.peerConn.removeTrack(sender);
        }
        // Empty the array
        senderArray.splice(0, senderArray.length);
        this.deleteFeedByStream(callFeed.stream);
    }

    private deleteAllFeeds(): void {
        for (const feed of this.feeds) {
            feed.dispose();
        }

        this.feeds = [];
        this.emit(CallEvent.FeedsChanged, this.feeds);
    }

    private deleteFeedByStream(stream: MediaStream): void {
        logger.debug(`Removing feed with stream id ${stream.id}`);

        const feed = this.getFeedByStreamId(stream.id);
        if (!feed) {
            logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`);
            return;
        }

        feed.dispose();
        this.feeds.splice(this.feeds.indexOf(feed), 1);
        this.emit(CallEvent.FeedsChanged, this.feeds);
    }

    // The typescript definitions have this type as 'any' :(
    public async getCurrentCallStats(): Promise<any[]> {
        if (this.callHasEnded()) {
            return this.callStatsAtEnd;
        }

        return this.collectCallStats();
    }

    private async collectCallStats(): Promise<any[]> {
        // This happens when the call fails before it starts.
        // For example when we fail to get capture sources
        if (!this.peerConn) return;

        const statsReport = await this.peerConn.getStats();
        const stats = [];
        statsReport.forEach(item => {
            stats.push(item);
        });

        return stats;
    }

    /**
     * Configure this call from an invite event. Used by MatrixClient.
     * @param {MatrixEvent} event The m.call.invite event
     */
    public async initWithInvite(event: MatrixEvent): Promise<void> {
        const invite = event.getContent<MCallInviteNegotiate>();
        this.direction = CallDirection.Inbound;

        // make sure we have valid turn creds. Unless something's gone wrong, it should
        // poll and keep the credentials valid so this should be instant.
        const haveTurnCreds = await this.client.checkTurnServers();
        if (!haveTurnCreds) {
            logger.warn("Failed to get TURN credentials! Proceeding with call anyway...");
        }

        const sdpStreamMetadata = invite[SDPStreamMetadataKey];
        if (sdpStreamMetadata) {
            this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
        } else {
            logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams");
        }

        this.peerConn = this.createPeerConnection();
        // we must set the party ID before await-ing on anything: the call event
        // handler will start giving us more call events (eg. candidates) so if
        // we haven't set the party ID, we'll ignore them.
        this.chooseOpponent(event);
        try {
            await this.peerConn.setRemoteDescription(invite.offer);
            await this.addBufferedIceCandidates();
        } catch (e) {
            logger.debug("Failed to set remote description", e);
            this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false);
            return;
        }

        const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream;

        // According to previous comments in this file, firefox at some point did not
        // add streams until media started arriving on them. Testing latest firefox
        // (81 at time of writing), this is no longer a problem, so let's do it the correct way.
        if (!remoteStream || remoteStream.getTracks().length === 0) {
            logger.error("No remote stream or no tracks after setting remote description!");
            this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false);
            return;
        }

        this.setState(CallState.Ringing);

        if (event.getLocalAge()) {
            setTimeout(() => {
                if (this.state == CallState.Ringing) {
                    logger.debug("Call invite has expired. Hanging up.");
                    this.hangupParty = CallParty.Remote; // effectively
                    this.setState(CallState.Ended);
                    this.stopAllMedia();
                    if (this.peerConn.signalingState != 'closed') {
                        this.peerConn.close();
                    }
                    this.emit(CallEvent.Hangup);
                }
            }, invite.lifetime - event.getLocalAge());
        }
    }

    /**
     * Configure this call from a hangup or reject event. Used by MatrixClient.
     * @param {MatrixEvent} event The m.call.hangup event
     */
    public initWithHangup(event: MatrixEvent): void {
        // perverse as it may seem, sometimes we want to instantiate a call with a
        // hangup message (because when getting the state of the room on load, events
        // come in reverse order and we want to remember that a call has been hung up)
        this.setState(CallState.Ended);
    }

    private shouldAnswerWithMediaType(
        wantedValue: boolean | undefined, valueOfTheOtherSide: boolean | undefined, type: "audio" | "video",
    ): boolean {
        if (wantedValue && !valueOfTheOtherSide) {
            // TODO: Figure out how to do this
            logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`);
            return false;
        } else if (
            !utils.isNullOrUndefined(wantedValue) &&
            wantedValue !== valueOfTheOtherSide &&
            !this.opponentSupportsSDPStreamMetadata()
        ) {
            logger.warn(
                `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` +
                `Answering with ${type}=${valueOfTheOtherSide}.`,
            );
            return valueOfTheOtherSide;
        }
        return wantedValue ?? valueOfTheOtherSide;
    }

    /**
     * Answer a call.
     */
    public async answer(audio?: boolean, video?: boolean): Promise<void> {
        if (this.inviteOrAnswerSent) return;
        // TODO: Figure out how to do this
        if (audio === false && video === false) throw new Error("You CANNOT answer a call without media");

        logger.debug(`Answering call ${this.callId}`);

        if (!this.localUsermediaStream && !this.waitForLocalAVStream) {
            const prevState = this.state;
            const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio");
            const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video");

            this.setState(CallState.WaitLocalMedia);
            this.waitForLocalAVStream = true;

            try {
                const stream = await this.client.getMediaHandler().getUserMediaStream(
                    answerWithAudio, answerWithVideo,
                );
                this.waitForLocalAVStream = false;
                const usermediaFeed = new CallFeed({
                    client: this.client,
                    roomId: this.roomId,
                    userId: this.client.getUserId(),
                    stream,
                    purpose: SDPStreamMetadataPurpose.Usermedia,
                    audioMuted: false,
                    videoMuted: false,
                });

                const feeds = [usermediaFeed];

                if (this.localScreensharingFeed) {
                    feeds.push(this.localScreensharingFeed);
                }

                this.answerWithCallFeeds(feeds);
            } catch (e) {
                if (answerWithVideo) {
                    // Try to answer without video
                    logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video");
                    this.setState(prevState);
                    this.waitForLocalAVStream = false;
                    await this.answer(answerWithAudio, false);
                } else {
                    this.getUserMediaFailed(e);
                    return;
                }
            }
        } else if (this.waitForLocalAVStream) {
            this.setState(CallState.WaitLocalMedia);
        }
    }

    public answerWithCallFeeds(callFeeds: CallFeed[]): void {
        if (this.inviteOrAnswerSent) return;

        this.gotCallFeedsForAnswer(callFeeds);
    }

    /**
     * Replace this call with a new call, e.g. for glare resolution. Used by
     * MatrixClient.
     * @param {MatrixCall} newCall The new call.
     */
    public replacedBy(newCall: MatrixCall): void {
        if (this.state === CallState.WaitLocalMedia) {
            logger.debug("Telling new call to wait for local media");
            newCall.waitForLocalAVStream = true;
        } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) {
            logger.debug("Handing local stream to new call");
            newCall.gotCallFeedsForAnswer(this.getLocalFeeds());
        }
        this.successor = newCall;
        this.emit(CallEvent.Replaced, newCall);
        this.hangup(CallErrorCode.Replaced, true);
    }

    /**
     * Hangup a call.
     * @param {string} reason The reason why the call is being hung up.
     * @param {boolean} suppressEvent True to suppress emitting an event.
     */
    public hangup(reason: CallErrorCode, suppressEvent: boolean): void {
        if (this.callHasEnded()) return;

        logger.debug("Ending call " + this.callId);
        this.terminate(CallParty.Local, reason, !suppressEvent);
        // We don't want to send hangup here if we didn't even get to sending an invite
        if (this.state === CallState.WaitLocalMedia) return;
        const content = {};
        // Don't send UserHangup reason to older clients
        if ((this.opponentVersion && this.opponentVersion !== 0) || reason !== CallErrorCode.UserHangup) {
            content["reason"] = reason;
        }
        this.sendVoipEvent(EventType.CallHangup, content);
    }

    /**
     * Reject a call
     * This used to be done by calling hangup, but is a separate method and protocol
     * event as of MSC2746.
     */
    public reject(): void {
        if (this.state !== CallState.Ringing) {
            throw Error("Call must be in 'ringing' state to reject!");
        }

        if (this.opponentVersion === 0) {
            logger.info(
                `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`,
            );
            this.hangup(CallErrorCode.UserHangup, true);
            return;
        }

        logger.debug("Rejecting call: " + this.callId);
        this.terminate(CallParty.Local, CallErrorCode.UserHangup, true);
        this.sendVoipEvent(EventType.CallReject, {});
    }

    /**
     * Adds an audio and/or video track - upgrades the call
     * @param {boolean} audio should add an audio track
     * @param {boolean} video should add an video track
     */
    private async upgradeCall(
        audio: boolean, video: boolean,
    ): Promise<void> {
        // We don't do call downgrades
        if (!audio && !video) return;
        if (!this.opponentSupportsSDPStreamMetadata()) return;

        try {
            const getAudio = audio || this.hasLocalUserMediaAudioTrack;
            const getVideo = video || this.hasLocalUserMediaVideoTrack;

            // updateLocalUsermediaStream() will take the tracks, use them as
            // replacement and throw the stream away, so it isn't reusable
            const stream = await this.client.getMediaHandler().getUserMediaStream(getAudio, getVideo, false);
            await this.updateLocalUsermediaStream(stream, audio, video);
        } catch (error) {
            logger.error("Failed to upgrade the call", error);
            this.emit(CallEvent.Error,
                new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error),
            );
        }
    }

    /**
     * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false
     * @returns {boolean} can screenshare
     */
    public opponentSupportsSDPStreamMetadata(): boolean {
        return Boolean(this.remoteSDPStreamMetadata);
    }

    /**
     * If there is a screensharing stream returns true, otherwise returns false
     * @returns {boolean} is screensharing
     */
    public isScreensharing(): boolean {
        return Boolean(this.localScreensharingStream);
    }

    /**
     * Starts/stops screensharing
     * @param enabled the desired screensharing state
     * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use
     * @returns {boolean} new screensharing state
     */
    public async setScreensharingEnabled(enabled: boolean, desktopCapturerSourceId?: string): Promise<boolean> {
        // Skip if there is nothing to do
        if (enabled && this.isScreensharing()) {
            logger.warn(`There is already a screensharing stream - there is nothing to do!`);
            return true;
        } else if (!enabled && !this.isScreensharing()) {
            logger.warn(`There already isn't a screensharing stream - there is nothing to do!`);
            return false;
        }

        // Fallback to replaceTrack()
        if (!this.opponentSupportsSDPStreamMetadata()) {
            return this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId);
        }

        logger.debug(`Set screensharing enabled? ${enabled}`);
        if (enabled) {
            try {
                const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId);
                if (!stream) return false;
                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
                return true;
            } catch (err) {
                logger.error("Failed to get screen-sharing stream:", err);
                return false;
            }
        } else {
            for (const sender of this.screensharingSenders) {
                this.peerConn.removeTrack(sender);
            }
            this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream);
            this.deleteFeedByStream(this.localScreensharingStream);
            return false;
        }
    }

    /**
     * Starts/stops screensharing
     * Should be used ONLY if the opponent doesn't support SDPStreamMetadata
     * @param enabled the desired screensharing state
     * @param {string} desktopCapturerSourceId optional id of the desktop capturer source to use
     * @returns {boolean} new screensharing state
     */
    private async setScreensharingEnabledWithoutMetadataSupport(
        enabled: boolean, desktopCapturerSourceId?: string,
    ): Promise<boolean> {
        logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`);
        if (enabled) {
            try {
                const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId);
                if (!stream) return false;

                const track = stream.getTracks().find((track) => {
                    return track.kind === "video";
                });
                const sender = this.usermediaSenders.find((sender) => {
                    return sender.track?.kind === "video";
                });
                sender.replaceTrack(track);

                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);

                return true;
            } catch (err) {
                logger.error("Failed to get screen-sharing stream:", err);
                return false;
            }
        } else {
            const track = this.localUsermediaStream.getTracks().find((track) => {
                return track.kind === "video";
            });
            const sender = this.usermediaSenders.find((sender) => {
                return sender.track?.kind === "video";
            });
            sender.replaceTrack(track);

            this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream);
            this.deleteFeedByStream(this.localScreensharingStream);

            return false;
        }
    }

    /**
     * Replaces/adds the tracks from the passed stream to the localUsermediaStream
     * @param {MediaStream} stream to use a replacement for the local usermedia stream
     */
    public async updateLocalUsermediaStream(
        stream: MediaStream, forceAudio = false, forceVideo = false,
    ): Promise<void> {
        const callFeed = this.localUsermediaFeed;
        const audioEnabled = forceAudio || (!callFeed.isAudioMuted() && !this.remoteOnHold);
        const videoEnabled = forceVideo || (!callFeed.isVideoMuted() && !this.remoteOnHold);
        setTracksEnabled(stream.getAudioTracks(), audioEnabled);
        setTracksEnabled(stream.getVideoTracks(), videoEnabled);

        // We want to keep the same stream id, so we replace the tracks rather than the whole stream
        for (const track of this.localUsermediaStream.getTracks()) {
            this.localUsermediaStream.removeTrack(track);
            track.stop();
        }
        for (const track of stream.getTracks()) {
            this.localUsermediaStream.addTrack(track);
        }

        const newSenders = [];

        for (const track of stream.getTracks()) {
            const oldSender = this.usermediaSenders.find((sender) => sender.track?.kind === track.kind);
            let newSender: RTCRtpSender;

            if (oldSender) {
                logger.info(
                    `Replacing track (` +
                    `id="${track.id}", ` +
                    `kind="${track.kind}", ` +
                    `streamId="${stream.id}", ` +
                    `streamPurpose="${callFeed.purpose}"` +
                    `) to peer connection`,
                );
                await oldSender.replaceTrack(track);
                newSender = oldSender;
            } else {
                logger.info(
                    `Adding track (` +
                    `id="${track.id}", ` +
                    `kind="${track.kind}", ` +
                    `streamId="${stream.id}", ` +
                    `streamPurpose="${callFeed.purpose}"` +
                    `) to peer connection`,
                );
                newSender = this.peerConn.addTrack(track, this.localUsermediaStream);
            }

            newSenders.push(newSender);
        }

        this.usermediaSenders = newSenders;
    }

    /**
     * Set whether our outbound video should be muted or not.
     * @param {boolean} muted True to mute the outbound video.
     * @returns the new mute state
     */
    public async setLocalVideoMuted(muted: boolean): Promise<boolean> {
        if (!await this.client.getMediaHandler().hasVideoDevice()) {
            return this.isLocalVideoMuted();
        }

        if (!this.hasLocalUserMediaVideoTrack && !muted) {
            await this.upgradeCall(false, true);
            return this.isLocalVideoMuted();
        }
        this.localUsermediaFeed?.setAudioVideoMuted(null, muted);
        this.updateMuteStatus();
        return this.isLocalVideoMuted();
    }

    /**
     * Check if local video is muted.
     *
     * If there are multiple video tracks, <i>all</i> of the tracks need to be muted
     * for this to return true. This means if there are no video tracks, this will
     * return true.
     * @return {Boolean} True if the local preview video is muted, else false
     * (including if the call is not set up yet).
     */
    public isLocalVideoMuted(): boolean {
        return this.localUsermediaFeed?.isVideoMuted();
    }

    /**
     * Set whether the microphone should be muted or not.
     * @param {boolean} muted True to mute the mic.
     * @returns the new mute state
     */
    public async setMicrophoneMuted(muted: boolean): Promise<boolean> {
        if (!await this.client.getMediaHandler().hasAudioDevice()) {
            return this.isMicrophoneMuted();
        }

        if (!this.hasLocalUserMediaAudioTrack && !muted) {
            await this.upgradeCall(true, false);
            return this.isMicrophoneMuted();
        }
        this.localUsermediaFeed?.setAudioVideoMuted(muted, null);
        this.updateMuteStatus();
        return this.isMicrophoneMuted();
    }

    /**
     * Check if the microphone is muted.
     *
     * If there are multiple audio tracks, <i>all</i> of the tracks need to be muted
     * for this to return true. This means if there are no audio tracks, this will
     * return true.
     * @return {Boolean} True if the mic is muted, else false (including if the call
     * is not set up yet).
     */
    public isMicrophoneMuted(): boolean {
        return this.localUsermediaFeed?.isAudioMuted();
    }

    /**
     * @returns true if we have put the party on the other side of the call on hold
     * (that is, we are signalling to them that we are not listening)
     */
    public isRemoteOnHold(): boolean {
        return this.remoteOnHold;
    }

    public setRemoteOnHold(onHold: boolean): void {
        if (this.isRemoteOnHold() === onHold) return;
        this.remoteOnHold = onHold;

        for (const transceiver of this.peerConn.getTransceivers()) {
            // We don't send hold music or anything so we're not actually
            // sending anything, but sendrecv is fairly standard for hold and
            // it makes it a lot easier to figure out who's put who on hold.
            transceiver.direction = onHold ? 'sendonly' : 'sendrecv';
        }
        this.updateMuteStatus();

        this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold);
    }

    /**
     * Indicates whether we are 'on hold' to the remote party (ie. if true,
     * they cannot hear us).
     * @returns true if the other party has put us on hold
     */
    public isLocalOnHold(): boolean {
        if (this.state !== CallState.Connected) return false;

        let callOnHold = true;

        // We consider a call to be on hold only if *all* the tracks are on hold
        // (is this the right thing to do?)
        for (const transceiver of this.peerConn.getTransceivers()) {
            const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection);

            if (!trackOnHold) callOnHold = false;
        }

        return callOnHold;
    }

    /**
     * Sends a DTMF digit to the other party
     * @param digit The digit (nb. string - '#' and '*' are dtmf too)
     */
    public sendDtmfDigit(digit: string): void {
        for (const sender of this.peerConn.getSenders()) {
            if (sender.track.kind === 'audio' && sender.dtmf) {
                sender.dtmf.insertDTMF(digit);
                return;
            }
        }

        throw new Error("Unable to find a track to send DTMF on");
    }

    private updateMuteStatus(): void {
        this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, {
            [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(),
        });

        const micShouldBeMuted = this.isMicrophoneMuted() || this.remoteOnHold;
        const vidShouldBeMuted = this.isLocalVideoMuted() || this.remoteOnHold;

        setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted);
        setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted);
    }

    private gotCallFeedsForInvite(callFeeds: CallFeed[]): void {
        if (this.successor) {
            this.successor.gotCallFeedsForAnswer(callFeeds);
            return;
        }
        if (this.callHasEnded()) {
            this.stopAllMedia();
            return;
        }

        for (const feed of callFeeds) {
            this.pushLocalFeed(feed);
        }

        this.setState(CallState.CreateOffer);

        logger.debug("gotUserMediaForInvite");
        // Now we wait for the negotiationneeded event
    }

    private async sendAnswer(): Promise<void> {
        const answerContent = {
            answer: {
                sdp: this.peerConn.localDescription.sdp,
                // type is now deprecated as of Matrix VoIP v1, but
                // required to still be sent for backwards compat
                type: this.peerConn.localDescription.type,
            },
            [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(),
        } as MCallAnswer;

        answerContent.capabilities = {
            'm.call.transferee': this.client.supportsCallTransfer,
            'm.call.dtmf': false,
        };

        // We have just taken the local description from the peerConn which will
        // contain all the local candidates added so far, so we can discard any candidates
        // we had queued up because they'll be in the answer.
        logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`);
        this.candidateSendQueue = [];

        try {
            await this.sendVoipEvent(EventType.CallAnswer, answerContent);
            // If this isn't the first time we've tried to send the answer,
            // we may have candidates queued up, so send them now.
            this.inviteOrAnswerSent = true;
        } catch (error) {
            // We've failed to answer: back to the ringing state
            this.setState(CallState.Ringing);
            this.client.cancelPendingEvent(error.event);

            let code = CallErrorCode.SendAnswer;
            let message = "Failed to send answer";
            if (error.name == 'UnknownDeviceError') {
                code = CallErrorCode.UnknownDevices;
                message = "Unknown devices present in the room";
            }
            this.emit(CallEvent.Error, new CallError(code, message, error));
            throw error;
        }

        // error handler re-throws so this won't happen on error, but
        // we don't want the same error handling on the candidate queue
        this.sendCandidateQueue();
    }

    private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise<void> {
        if (this.callHasEnded()) return;

        this.waitForLocalAVStream = false;

        for (const feed of callFeeds) {
            this.pushLocalFeed(feed);
        }

        this.setState(CallState.CreateAnswer);

        let myAnswer;
        try {
            this.getRidOfRTXCodecs();
            myAnswer = await this.peerConn.createAnswer();
        } catch (err) {
            logger.debug("Failed to create answer: ", err);
            this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true);
            return;
        }

        try {
            await this.peerConn.setLocalDescription(myAnswer);
            this.setState(CallState.Connecting);

            // Allow a short time for initial candidates to be gathered
            await new Promise(resolve => {
                setTimeout(resolve, 200);
            });

            this.sendAnswer();
        } catch (err) {
            logger.debug("Error setting local description!", err);
            this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
            return;
        }
    }

    /**
     * Internal
     * @param {Object} event
     */
    private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): Promise<void> => {
        if (event.candidate) {
            logger.debug(
                "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " +
                event.candidate.candidate,
            );

            if (this.callHasEnded()) return;

            // As with the offer, note we need to make a copy of this object, not
            // pass the original: that broke in Chrome ~m43.
            if (event.candidate.candidate !== '' || !this.sentEndOfCandidates) {
                this.queueCandidate(event.candidate);

                if (event.candidate.candidate === '') this.sentEndOfCandidates = true;
            }
        }
    };

    private onIceGatheringStateChange = (event: Event): void => {
        logger.debug("ice