fcr-core/lib-es/room-control/stt-control/index.js

Core APIs for building online scenes

import "core-js/modules/esnext.function.metadata.js";
import "core-js/modules/esnext.symbol.metadata.js";
let _initProto, _startCaptionDecs, _startTranscribingDecs, _updateSourceLanguageDecs, _subscribeTranscribingLanguageDecs;
import "core-js/modules/es.array.push.js";
import "core-js/modules/esnext.iterator.constructor.js";
import "core-js/modules/esnext.iterator.filter.js";
import "core-js/modules/esnext.iterator.map.js";
import "core-js/modules/esnext.map.delete-all.js";
import "core-js/modules/esnext.map.emplace.js";
import "core-js/modules/esnext.map.every.js";
import "core-js/modules/esnext.map.filter.js";
import "core-js/modules/esnext.map.find.js";
import "core-js/modules/esnext.map.find-key.js";
import "core-js/modules/esnext.map.includes.js";
import "core-js/modules/esnext.map.key-of.js";
import "core-js/modules/esnext.map.map-keys.js";
import "core-js/modules/esnext.map.map-values.js";
import "core-js/modules/esnext.map.merge.js";
import "core-js/modules/esnext.map.reduce.js";
import "core-js/modules/esnext.map.some.js";
import "core-js/modules/esnext.map.update.js";
function _applyDecs(e, t, r, n, o, a) { function i(e, t, r) { return function (n, o) { return r && r(n), e[t].call(n, o); }; } function c(e, t) { for (var r = 0; r < e.length; r++) e[r].call(t); return t; } function s(e, t, r, n) { if ("function" != typeof e && (n || void 0 !== e)) throw new TypeError(t + " must " + (r || "be") + " a function" + (n ? "" : " or undefined")); return e; } function applyDec(e, t, r, n, o, a, c, u, l, f, p, d, h) { function m(e) { if (!h(e)) throw new TypeError("Attempted to access private element on non-instance"); } var y, v = t[0], g = t[3], b = !u; if (!b) { r || Array.isArray(v) || (v = [v]); var w = {}, S = [], A = 3 === o ? "get" : 4 === o || d ? "set" : "value"; f ? (p || d ? w = { get: _setFunctionName(function () { return g(this); }, n, "get"), set: function (e) { t[4](this, e); } } : w[A] = g, p || _setFunctionName(w[A], n, 2 === o ? "" : A)) : p || (w = Object.getOwnPropertyDescriptor(e, n)); } for (var P = e, j = v.length - 1; j >= 0; j -= r ? 2 : 1) { var D = v[j], E = r ? v[j - 1] : void 0, I = {}, O = { kind: ["field", "accessor", "method", "getter", "setter", "class"][o], name: n, metadata: a, addInitializer: function (e, t) { if (e.v) throw Error("attempted to call addInitializer after decoration was finished"); s(t, "An initializer", "be", !0), c.push(t); }.bind(null, I) }; try { if (b) (y = s(D.call(E, P, O), "class decorators", "return")) && (P = y);else { var k, F; O.static = l, O.private = f, f ? 2 === o ? k = function (e) { return m(e), w.value; } : (o < 4 && (k = i(w, "get", m)), 3 !== o && (F = i(w, "set", m))) : (k = function (e) { return e[n]; }, (o < 2 || 4 === o) && (F = function (e, t) { e[n] = t; })); var N = O.access = { has: f ? h.bind() : function (e) { return n in e; } }; if (k && (N.get = k), F && (N.set = F), P = D.call(E, d ? { get: w.get, set: w.set } : w[A], O), d) { if ("object" == typeof P && P) (y = s(P.get, "accessor.get")) && (w.get = y), (y = s(P.set, "accessor.set")) && (w.set = y), (y = s(P.init, "accessor.init")) && S.push(y);else if (void 0 !== P) throw new TypeError("accessor decorators must return an object with get, set, or init properties or void 0"); } else s(P, (p ? "field" : "method") + " decorators", "return") && (p ? S.push(P) : w[A] = P); } } finally { I.v = !0; } } return (p || d) && u.push(function (e, t) { for (var r = S.length - 1; r >= 0; r--) t = S[r].call(e, t); return t; }), p || b || (f ? d ? u.push(i(w, "get"), i(w, "set")) : u.push(2 === o ? w[A] : i.call.bind(w[A])) : Object.defineProperty(e, n, w)), P; } function u(e, t) { return Object.defineProperty(e, Symbol.metadata || Symbol.for("Symbol.metadata"), { configurable: !0, enumerable: !0, value: t }); } if (arguments.length >= 6) var l = a[Symbol.metadata || Symbol.for("Symbol.metadata")]; var f = Object.create(null == l ? null : l), p = function (e, t, r, n) { var o, a, i = [], s = function (t) { return _checkInRHS(t) === e; }, u = new Map(); function l(e) { e && i.push(c.bind(null, e)); } for (var f = 0; f < t.length; f++) { var p = t[f]; if (Array.isArray(p)) { var d = p[1], h = p[2], m = p.length > 3, y = 16 & d, v = !!(8 & d), g = 0 == (d &= 7), b = h + "/" + v; if (!g && !m) { var w = u.get(b); if (!0 === w || 3 === w && 4 !== d || 4 === w && 3 !== d) throw Error("Attempted to decorate a public method/accessor that has the same name as a previously decorated public method/accessor. This is not currently supported by the decorators plugin. Property name was: " + h); u.set(b, !(d > 2) || d); } applyDec(v ? e : e.prototype, p, y, m ? "#" + h : _toPropertyKey(h), d, n, v ? a = a || [] : o = o || [], i, v, m, g, 1 === d, v && m ? s : r); } } return l(o), l(a), i; }(e, t, o, f); return r.length || u(e, f), { e: p, get c() { var t = []; return r.length && [u(applyDec(e, [r], n, e.name, 5, f, t), f), c.bind(null, t, e)]; } }; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != typeof t || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
function _setFunctionName(e, t, n) { "symbol" == typeof t && (t = (t = t.description) ? "[" + t + "]" : ""); try { Object.defineProperty(e, "name", { configurable: !0, value: n ? n + " " + t : t }); } catch (e) {} return e; }
function _checkInRHS(e) { if (Object(e) !== e) throw TypeError("right-hand side of 'in' should be an object, got " + (null !== e ? typeof e : "null")); return e; }
import { AgoraObservable, trace, deCompressGzipToJson, merge } from '../../imports';
import { createLogger, generateLogObserver } from '../../utilities/logger';
import { fcrLanguageSchema, fcrLanguageConfigSchema } from '../../schema';
import { FcrCaptionState, FcrTranscribingState } from './type';
import { FcrErrorModuleCode, handleRequestError } from '../../utilities/error';
import { FcrLanguage } from '../../type';
import validateParams from '../../utilities/validate-params';
import { convertRteUserToFcrUser } from '../../utilities/user';
import { FcrWidgetCauseCmd, FcrWidgetUuid } from '../type';
/**
 * @internal
 */
export class FcrSttControlImpl {
  static {
    [_initProto] = _applyDecs(this, [[_startCaptionDecs, 2, "startCaption"], [trace, 2, "stopCaption"], [_startTranscribingDecs, 2, "startTranscribing"], [trace, 2, "stopTranscribing"], [_updateSourceLanguageDecs, 2, "updateSourceLanguage"], [_subscribeTranscribingLanguageDecs, 2, "subscribeTranscribingLanguage"], [trace, 2, "unsubscribeTranscribingLanguage"], [trace, 2, "release"]], []).e;
  }
  //@internal
  [(_startCaptionDecs = [trace, validateParams(fcrLanguageConfigSchema)], _startTranscribingDecs = [trace, validateParams(fcrLanguageConfigSchema)], _updateSourceLanguageDecs = [trace, validateParams(fcrLanguageSchema)], _subscribeTranscribingLanguageDecs = [trace, validateParams(fcrLanguageSchema)], "logger")] = (_initProto(this), createLogger({
    prefix: 'FcrUserControlImpl'
  }));

  //@internal
  _observable = new AgoraObservable();

  //@internal

  _cacheMaxLimit = 10;
  _sttEventWrapperListMap = new Map();

  //@internal
  _sceneObserver = {
    onStreamMessageReceived: async (sceneId, data) => {
      const sttUids = this._scene.getScenePropertiesByKeyPath('widgets.stt.extra.pubBotUids');
      const isSttBot = sttUids.includes(Number(data.streamId));
      if (!isSttBot) {
        return;
      }
      const sttStreamMessageData = await deCompressGzipToJson(data.payload);
      this._handleSttStreamMessageData(sttStreamMessageData, sceneId);
    },
    onUserPropertiesUpdated: (sceneId, event) => {
      if (event.cause && event.cause.data) {
        const causeData = event.cause.data;
        if (causeData.widgetUuid == FcrWidgetUuid.STT && causeData.widgetCause && causeData.widgetCause.cmd == 3450) {
          const localUserId = this._scene.localUser.getLocalUserId();
          if (event.operatorUser.userId !== localUserId) {
            return;
          }
          const operatorUser = convertRteUserToFcrUser(event.operatorUser, this._roomCache);
          if (causeData.widgetCause.reason === 0) {
            const isTranscribingStopped = this._getTranscribingState() === FcrTranscribingState.STOPPED;
            if (isTranscribingStopped) {
              this._sttEventWrapperListMap.clear();
            }
            this._observable.notifyObservers('onLocalCaptionStateUpdated', sceneId, FcrCaptionState.STOPPED, operatorUser);
          }
          if (causeData.widgetCause.reason === 1) {
            this._observable.notifyObservers('onLocalCaptionStateUpdated', sceneId, FcrCaptionState.STARTED, operatorUser);
          }
        }
      }
    },
    onScenePropertiesUpdated: (sceneId, event) => {
      if (event.cause && event.cause.data) {
        const causeData = event.cause.data;
        if (!causeData.widgetCause) {
          return;
        }
        if (causeData.widgetUuid == FcrWidgetUuid.TRANSCRIBE && causeData.widgetCause.cmd == FcrWidgetCauseCmd.TRANSCRIBE) {
          const operatorUser = convertRteUserToFcrUser(event.operatorUser, this._roomCache);
          if (causeData.widgetCause.reason === 0) {
            const isCaptionStopped = this._getLocalCaptionState() === FcrCaptionState.STOPPED;
            if (isCaptionStopped) {
              this._sttEventWrapperListMap.clear();
            }
            this._observable.notifyObservers('onTranscribingStateUpdated', sceneId, FcrTranscribingState.STOPPED, operatorUser);
          }
          if (causeData.widgetCause.reason === 1) {
            this._observable.notifyObservers('onTranscribingStateUpdated', sceneId, FcrTranscribingState.STARTED, operatorUser);
          }
        }
        if (causeData.widgetUuid == FcrWidgetUuid.STT && causeData.widgetCause.cmd == FcrWidgetCauseCmd.STT) {
          let operatorUser;
          if (event.operatorUser) {
            operatorUser = convertRteUserToFcrUser(event.operatorUser, this._roomCache);
          }
          if (causeData.widgetCause.reason === 2) {
            this._observable.notifyObservers('onSourceLanguageUpdated', sceneId, operatorUser);
          }
        }
      }
    }
  };
  constructor(_scene, _api, _sharedCache) {
    this._scene = _scene;
    this._api = _api;
    this._sharedCache = _sharedCache;
    this._roomCache = _sharedCache.getRoomCache(this._scene.sceneId);
    this._scene.addObserver(this._sceneObserver);
    this._addLogObserver();
  }
  startCaption(config) {
    return handleRequestError(() => this._api.startCaption({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId(),
      config
    }), FcrErrorModuleCode.ROOM_STT, 'start caption failed');
  }
  stopCaption() {
    return handleRequestError(() => this._api.stopCaption({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId()
    }), FcrErrorModuleCode.ROOM_STT, 'stop caption failed');
  }
  startTranscribing(config) {
    return handleRequestError(() => this._api.startTranscribing({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId(),
      config
    }), FcrErrorModuleCode.ROOM_STT, 'start transcribing failed');
  }
  stopTranscribing() {
    return handleRequestError(() => this._api.stopTranscribing({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId()
    }), FcrErrorModuleCode.ROOM_STT, 'stop transcribing failed');
  }
  updateSourceLanguage(language) {
    return handleRequestError(() => this._api.updateSourceLanguage({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId(),
      language
    }), FcrErrorModuleCode.ROOM_STT, 'update source language failed');
  }
  subscribeTranscribingLanguage(language) {
    return handleRequestError(() => this._api.subscribeTranscribingLanguage({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId(),
      language
    }), FcrErrorModuleCode.ROOM_STT, 'subscribe transcribing language failed');
  }
  unsubscribeTranscribingLanguage() {
    return handleRequestError(() => this._api.unsubscribeTranscribingLanguage({
      roomUuid: this._scene.sceneId,
      userUuid: this._scene.localUser.getLocalUserId()
    }), FcrErrorModuleCode.ROOM_STT, 'unsubscribe transcribing language failed');
  }
  getLocalCaptionState() {
    return this._getLocalCaptionState();
  }
  getTranscribingState() {
    return this._getTranscribingState();
  }
  getSourceLanguage() {
    const transcribingState = this._getTranscribingState();
    if (transcribingState === FcrTranscribingState.STOPPED) {
      return null;
    }
    return this._scene.getScenePropertiesByKeyPath('widgets.stt.extra.speechLanguage');
  }
  getConfig() {
    const sourceLanguages = this._scene.getScenePropertiesByKeyPath('widgets.stt.extra.sourceLanguages');
    const translateLanguages = this._scene.getScenePropertiesByKeyPath('widgets.stt.extra.translateLanguages');
    const cacheMaxLimit = this._scene.getScenePropertiesByKeyPath('widgets.stt.extra.maxLimit');
    if (!sourceLanguages || !translateLanguages || !cacheMaxLimit) {
      return null;
    }
    return {
      sourceLanguages,
      translateLanguages,
      cacheMaxLimit
    };
  }
  addObserver(observer) {
    this._observable.addObserver(observer);
  }
  removeObserver(observer) {
    this._observable.removeObserver(observer);
  }
  release() {
    this._sttEventWrapperListMap.clear();
    this._scene.removeObserver(this._sceneObserver);
  }
  _getLocalCaptionState() {
    const captionState = this._scene.getUserPropertiesByKeyPath('widgets.stt.caption', this._scene.localUser.getLocalUserId());
    return captionState;
  }
  _getTranscribingState() {
    return this._scene.getScenePropertiesByKeyPath('widgets.transcribe.state');
  }
  _handleSttStreamMessageData(sttStreamMessageData, sceneId) {
    const codeMap = {
      'zh-CN': FcrLanguage.ZH,
      'en-US': FcrLanguage.EN,
      'ja-JP': FcrLanguage.JP,
      'de-DE': FcrLanguage.DE,
      'fr-FR': FcrLanguage.FR,
      'pt-PT': FcrLanguage.PT,
      'es-ES': FcrLanguage.ES
    };
    const transcript = sttStreamMessageData.transcript;
    const translation = sttStreamMessageData.translation;
    const messageItem = transcript || translation;
    const uid = messageItem?.uid;
    if (!uid) {
      this.logger.error('parseSTT failed, uid is null');
      return null;
    }
    const owner = this._scene.getStreamByStreamId(uid.toString())?.owner;
    if (!owner) {
      this.logger.error('parseSTT failed, owner is null');
      return null;
    }
    const userId = owner.userId;
    if (!userId) {
      this.logger.error('parseSTT failed, userId is null');
      return null;
    }
    let sttEventWrapperList = this._sttEventWrapperListMap.get(userId);
    let sttEventWrapper = null;
    if (transcript) {
      const sentenceId = transcript.uid + '-' + transcript.textTs;
      const formatTranscript = {
        id: sentenceId,
        text: transcript.text,
        language: codeMap[transcript.language],
        owner: convertRteUserToFcrUser(owner, this._roomCache),
        timestamp: transcript.textTs,
        isFinal: transcript.isFinal
      };
      if (!sttEventWrapperList || sttEventWrapperList.length === 0) {
        sttEventWrapper = {
          startTs: 0,
          endTs: transcript.textTs,
          event: {
            transcript: formatTranscript,
            translations: new Map()
          }
        };
        sttEventWrapperList = [sttEventWrapper];
        this._observable.notifyObservers('onSttEventReceived', sceneId, sttEventWrapper.event);
      } else {
        const lastSttEventWrapper = sttEventWrapperList[sttEventWrapperList.length - 1];
        // add
        if (lastSttEventWrapper.event.transcript.isFinal == true) {
          sttEventWrapper = {
            startTs: lastSttEventWrapper.endTs,
            endTs: transcript.textTs,
            event: {
              transcript: formatTranscript,
              translations: new Map()
            }
          };
          sttEventWrapperList.push(sttEventWrapper);
          // 超过 10 条，移除 oldest
          if (sttEventWrapperList.length > this._cacheMaxLimit) {
            // remove oldest
            sttEventWrapperList.shift();
          }
          this._observable.notifyObservers('onSttEventReceived', sceneId, sttEventWrapper.event);
        } else {
          if (transcript.textTs >= lastSttEventWrapper.endTs) {
            // update
            sttEventWrapperList[sttEventWrapperList.length - 1] = {
              startTs: lastSttEventWrapper.startTs,
              endTs: transcript.textTs,
              event: merge(lastSttEventWrapper.event, {
                transcript: {
                  ...formatTranscript,
                  id: lastSttEventWrapper.event.transcript.id
                }
              })
            };
            this._observable.notifyObservers('onSttEventReceived', sceneId, sttEventWrapperList[sttEventWrapperList.length - 1].event);
          }
        }
      }
    }
    if (translation) {
      // 获取缓存句子
      sttEventWrapperList = sttEventWrapperList?.map(wrapper => {
        const language = codeMap[translation.results0.language];
        const formatTranslation = {
          id: wrapper.event.transcript.id,
          language,
          text: translation.results0.texts.join(''),
          timestamp: translation.textTs,
          isFinal: translation.isFinal,
          owner: convertRteUserToFcrUser(owner, this._roomCache)
        };
        if (translation.textTs > wrapper.startTs && translation.textTs <= wrapper.endTs) {
          const transInfo = wrapper.event.translations.get(language);
          if (!transInfo) {
            wrapper.event.translations.set(formatTranslation.language, formatTranslation);
            this._observable.notifyObservers('onSttEventReceived', sceneId, wrapper.event);
          } else if (translation.textTs >= transInfo.timestamp) {
            wrapper.event.translations.set(formatTranslation.language, {
              ...formatTranslation,
              id: wrapper.event.transcript.id
            });
            this._observable.notifyObservers('onSttEventReceived', sceneId, wrapper.event);
          } else if (transInfo.isFinal == true && wrapper.event.transcript.isFinal == true) {
            return null;
          }
        }
        return wrapper;
      }).filter(wrapper => wrapper !== null);
    }
    if (sttEventWrapperList) {
      this._sttEventWrapperListMap.set(userId, sttEventWrapperList);
    }
  }
  _addLogObserver() {
    this.addObserver(generateLogObserver(this.logger, ['onSttEventReceived', 'onLocalCaptionStateUpdated', 'onTranscribingStateUpdated', 'onSourceLanguageUpdated']));
  }
}