vroom-web-sdk-beta/src/plugins/vroom-video.plugin.ts

VROOM SDK (beta) by True Virtual World

import VroomSession from '../session/vroom.session'
import { iceServer, VROOM_COMMAND_STATUS, VROOM_PLUGIN_TYPE, BITRATE } from '../constants'
import { get, concat, flatten, isEmpty } from 'lodash'
import VroomParticipant from '../types/publisher'
import {
  StartRoomMessage,
  DetachPluginMessage,
  AttachPluginMessage,
  JoinRoomAsPublisher,
  JoinRoomAsSubscriber,
  UpdateSubscriber,
  SendOffer,
  MuteAudio,
  MuteVideo,
  TrickleMessage
} from '../types/vroomRequest'

class VroomVideoPlugin {

  // id
  sessionId: number
  handleId?: number
  opaqueId: string
  keepAliveTimeoutId: any

  // state plugin
  isWebRtcUp: boolean = false
  connected: boolean = true
  isRemoteDescriptionSet: boolean = false
  delegate?: VroomVideoPluginDelegate
  roomId?: number
  userId?: number
  subscriberFirstJoined = false
  stream!: MediaStream
  videoTrack?: MediaStreamTrack
  audioTrack?: MediaStreamTrack

  // dependencies
  peerConnection!: RTCPeerConnection
  cachedCandidates: RTCIceCandidate[] = []
  session: VroomSession

  constructor(session: VroomSession, sessionId: number) {
    this.sessionId  = sessionId
    this.opaqueId   = 'xxx'

    this.session    = session

    this.initPeerConnection()
  }

  private initPeerConnection() {
    this.peerConnection = new RTCPeerConnection({
      iceServers: iceServer,
    })

    this.peerConnection.onicecandidate = (event) => {
      if (!event.candidate || event.candidate?.candidate.indexOf('endOfCandidates') < 0) {
        // TODO document why this block is empty
      } else {
        this.send(new TrickleMessage({
          session_id: this.sessionId,
          handle_id: this.handleId,
          candidate: event.candidate,
        }))
      }
    }
  }

  async createOffer(bitrate: number, audio: boolean, video: boolean): Promise<void> {

    const offerObj = {
      offerToReceiveAudio: true,
      offerToReceiveVideo: true,
    }

    const _offer = await this.peerConnection.createOffer(offerObj)

    await this.peerConnection.setLocalDescription(_offer)

    const offerResponse: any = await this.send(new SendOffer({
      session_id: this.sessionId,
      handle_id: this.handleId,
      body: {
        request: 'configure',
        audio,
        video,
        bitrate
      },
      jsep: _offer
    }))

    await this.peerConnection.setRemoteDescription(
      new RTCSessionDescription({
        sdp: offerResponse.jsep.sdp,
        type: offerResponse.jsep.type,
      }),
    )

    this.isRemoteDescriptionSet = true

    this.cachedCandidates.forEach((candidate: RTCIceCandidate) => {
      this.peerConnection.addIceCandidate(candidate)
    })

    this.cachedCandidates = [] as RTCIceCandidate[]
  }

  async createAnswer(jsep: any) {
    await this.peerConnection.setRemoteDescription(
      new RTCSessionDescription({
        sdp: jsep.sdp,
        type: jsep.type,
      }),
    )

    this.isRemoteDescriptionSet = true

    this.cachedCandidates.forEach(async (candidate: RTCIceCandidate) => {
      this.peerConnection.addIceCandidate(candidate)
    })

    this.cachedCandidates = [] as RTCIceCandidate[]

    const answerRes = await this.peerConnection.createAnswer({
      offerToReceiveAudio: true,
      offerToReceiveVideo: true,
    })

    await this.peerConnection.setLocalDescription(answerRes)

    return answerRes
  }


  async close() {
    this.peerConnection.close()
  }

  async onMessage(message: any)  {
    const { janus } = message

    switch(janus){
      case VROOM_COMMAND_STATUS.TRICKLE:
        this.handleTrickleMessage(message)
        break
      case VROOM_COMMAND_STATUS.EVENT:
        this.handleEvent(message)
        break
    }

    if (message['jsep']) {
      const answerRes = await this.createAnswer(message['jsep'])
      await this.send(new StartRoomMessage({
        session_id: this.sessionId,
        handle_id: this.handleId,
        room: this.roomId,
        jsep: answerRes
      }))

    }
  }

  private handleEvent(message: any) {
    const parseMessage = this.retrieveVroomMessageEnumFromData(message)

    const { type, data } = parseMessage

    switch(type) {
      case VROOM_PLUGIN_TYPE.S_JOIN:
        if (data?.length > 0) {
          data.forEach((p: any) => this.delegate?.onParticipantJoined(new VroomParticipant(p)))
        }

        break
      case VROOM_PLUGIN_TYPE.LEAVING:
        this.delegate?.onParticipantLeave({ id: data })
        break
      case VROOM_PLUGIN_TYPE.MUTE_AUDIO:
        this.delegate?.onParticipantMuteAudio(data)
        break
      case VROOM_PLUGIN_TYPE.MUTE_VIDEO:
        this.delegate?.onParticipantMuteVideo(data)
        break
      case VROOM_PLUGIN_TYPE.UPDATED:
        this.delegate?.onStreamUpdated(data)
    }
  }

  private handleTrickleMessage(message: any) {
    if (this.isRemoteDescriptionSet && message.candidate.sdpMid) {
      if (!message?.candidate?.completed) {
        this.peerConnection.addIceCandidate(
          new RTCIceCandidate({
            candidate: message.candidate.candidate,
            sdpMid: message.candidate.sdpMid,
            sdpMLineIndex: message.candidate.sdpMLineIndex,
          }),
        )
      }
    }
    if (!message?.candidate.completed) {
      this.cachedCandidates.push(
        new RTCIceCandidate({
          candidate: message.candidate.candidate,
          sdpMid: message.candidate.sdpMid,
          sdpMLineIndex: message.candidate.sdpMLineIndex,
        }),
      )
    }
  }

  /**
   *
   * @param data Object
   * @returns { type: String, ... }
   */
  retrieveVroomMessageEnumFromData(data: { [id: string]: any }) {
    if (!data) return { type: VROOM_PLUGIN_TYPE.UNKNOW }

    if (get(data, 'leaving', false) || get(data, 'plugindata.data.leaving')) {
      const leavingPublisherID = get(data, 'plugindata.data.leaving')
      return { type: VROOM_PLUGIN_TYPE.LEAVING, data: leavingPublisherID }
    }

    switch(get(data, 'plugindata.data.videoroom')) {
      case VROOM_PLUGIN_TYPE.EVENT: {
        const plugindata = get(data, 'plugindata.data')

        const publisher = get(plugindata, 'publishers', [])
        const attendees = get(plugindata, 'attendees', [])
        const concatParticipant = concat(publisher, attendees)

        return { type: VROOM_PLUGIN_TYPE.S_JOIN, data: concatParticipant }
      }

      case VROOM_PLUGIN_TYPE.S_JOIN:
        return { type: VROOM_PLUGIN_TYPE.S_JOIN, data: [ get(data, 'plugindata.data') ] }

      case VROOM_PLUGIN_TYPE.MUTE_AUDIO:
        return { type: VROOM_PLUGIN_TYPE.MUTE_AUDIO, data: get(data, 'plugindata.data') }

      case VROOM_PLUGIN_TYPE.MUTE_VIDEO:
        return { type: VROOM_PLUGIN_TYPE.MUTE_VIDEO, data: get(data, 'plugindata.data') }

      case VROOM_PLUGIN_TYPE.UPDATED:
        return { type: VROOM_PLUGIN_TYPE.UPDATED, data: get(data, 'plugindata.data.streams', []) }
      default:
        return { type: VROOM_PLUGIN_TYPE.UNKNOW }
    }
  }

  async attachPlugin() {
    const request = new AttachPluginMessage({
      session_id: this.sessionId,
      opaque: this.opaqueId,
    })

    const result: any = await this.send(request)
    this.handleId = result.data.id
    this.session.attachPlugin(this)
  }

  async detachPlugin(withCloseSession = true) {
    const request = new DetachPluginMessage({
      session_id: this.sessionId,
      handle_id: this.handleId
    })

    await this.send(request)

    this.peerConnection.close()
    this.session.deletePlugin(this)

    if (withCloseSession) {
      this.connected = false
      await this.session.destroy(this.sessionId)
    }
  }

  private async send(request: any) {
    return this.session.sendAsync(request)
  }

  public setStream(stream: MediaStream) {
    this.stream = stream
  }

  public async joinRoom(
    stream: MediaStream,
    roomId: number,
    display: string,
    muteAudio = false,
    muteVideo = false,
  ){
    try {
      this.stream = stream
      this.roomId = roomId

      this.videoTrack = stream.getVideoTracks()[0]
      this.audioTrack = stream.getAudioTracks()[0]

      const joinRequest = new JoinRoomAsPublisher({
        room: roomId,
        display,
        audio: muteAudio,
        video: muteVideo,
        session_id: this.sessionId,
        handle_id: this.handleId
      })

      let joinResponse: any = await this.send(joinRequest)

      if (joinResponse.janus === 'event' && get(joinResponse, 'plugindata.data', false)) {
        const data = joinResponse.plugindata.data

        if (data.videoroom === 'joined') {
          // TODO: handle if mute is true
          // process offer if mute is false
          stream.getTracks().forEach((track: MediaStreamTrack) => {
            this.peerConnection.addTrack(track)
          })

          await this.createOffer(BITRATE, !muteAudio, !muteVideo)

          this.delegate?.onJoinRoomComplete(data)
          return
        }
        // emit join failure
        this.delegate?.onJoinRoomFailure(data)
      }
    } catch (e) {
      // emit join failure with error
      this.delegate?.onJoinRoomFailure(e)
    }
  }

  async receive(publisher: any[], privateId: number, isSubscribe = true) {
    let response: any = null

    const keyOption = isSubscribe ? 'subscribe' : 'unsubscribe'
    const streams = flatten(
      publisher.map((p: any) => {
        if (!isEmpty(p?.streams)) {
          return p.streams?.map((s: any) => {
            if (s?.type === 'video') {
              return {
                feed: p?.id,
                mid: s?.mid,
                substream: 1, // medium
              }
            }
            return {
              feed: p?.id,
              mid: s?.mid,
            }
          })

        } else if (!isEmpty(p?.feed_id) && !isEmpty(p?.mid)) {
          const withVideoOption = p.type === 'video' ? { substream: 1 } : {}
          return {
            feed: p.feed_id,
            mid: p.mid,
            ...withVideoOption
          }
        }
      }),
    )

    if (!this.subscriberFirstJoined) {
      this.subscriberFirstJoined = true

      const request = new JoinRoomAsSubscriber({
        session_id: this.sessionId,
        handle_id: this.handleId,
        room: this.roomId,
        streams,
        private_id: privateId,
      })

      response = await this.send(request)

    } else {
      try {
        const updateObj: any = {
          request: 'update',
          [keyOption]: streams,
        }

        response = await this.send(new UpdateSubscriber({
          session_id: this.sessionId,
          handle_id: this.handleId,
          body: updateObj,
        }))
      } catch (e) {
        console.error('update error ', e)
      }
    }

    if (!isEmpty(get(response, 'plugindata.data.streams'))) {
      const dataStreams = get(response, 'plugindata.data.streams')
      this.delegate?.onStreamUpdated(dataStreams)
    }

    if (!isEmpty(response?.jsep)) {
      const answerRes = await this.createAnswer(response?.jsep)
      const startRequest = new StartRoomMessage({
        session_id: this.sessionId,
        handle_id: this.handleId,
        room: this.roomId,
        jsep: answerRes
      })

      await this.send(startRequest)
    }
  }

  public async sendMuteAudio(mute: boolean) {
    await this.session.sendAsync(new MuteAudio({
      session_id: this.sessionId,
      handle_id: this.handleId,
      room: this.roomId,
      mute,
      id: this.userId
    }))
  }

  public async sendMuteVideo(mute: boolean) {
    await this.session.sendAsync(new MuteVideo({
      session_id: this.sessionId,
      handle_id: this.handleId,
      room: this.roomId,
      mute,
      id: this.userId
    }))
  }

  public isAudioMuted() {
    return !this.stream.getAudioTracks()[0].enabled
  }

  public muteAudio() {
    this.stream.getAudioTracks()[0].enabled = false
  }

  public unmuteAudio() {
    this.stream.getAudioTracks()[0].enabled = true
  }

  public isVideoMuted() {
    return !this.stream.getVideoTracks()[0].enabled
  }

  public muteVideo() {
    this.stream.getVideoTracks()[0].enabled = false
  }

  public unmuteVideo() {
    this.stream.getVideoTracks()[0].enabled = true
  }
}

export interface VroomVideoPluginDelegate {
  onJoinRoomComplete: (data: any) => {}
  onJoinRoomFailure: (error: any) => {}

  onStreamUpdated: (streams: any[]) => {}

  onParticipantJoined: (participant: VroomParticipant) => {}
  onParticipantLeave: (participant: VroomParticipant) => {}
  onParticipantMuteAudio: (participant: any) => {}
  onParticipantMuteVideo: (participant: any) => {}
}

export default VroomVideoPlugin