@skyway-sdk/sfu-bot/dist/connection/sender.js

The official Next Generation JavaScript SDK for SkyWay

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Sender = void 0;
const common_1 = require("@skyway-sdk/common");
const core_1 = require("@skyway-sdk/core");
const isEqual_1 = __importDefault(require("lodash/isEqual"));
const errors_1 = require("../errors");
const forwarding_1 = require("../forwarding");
const util_1 = require("../util");
const log = new common_1.Logger('packages/sfu-bot/src/connection/sender.ts');
class Sender {
    constructor(publication, channel, _api, _transportRepository, _localPerson, _bot, _iceManager, _context) {
        this.publication = publication;
        this.channel = channel;
        this._api = _api;
        this._transportRepository = _transportRepository;
        this._localPerson = _localPerson;
        this._bot = _bot;
        this._iceManager = _iceManager;
        this._context = _context;
        this._disposer = new common_1.EventDisposer();
        this._connectionState = 'new';
        this.onConnectionStateChanged = new common_1.Event();
        this.closed = false;
        this.sendSubscriptionStatsReportTimer = null;
        this._waitingSendSubscriptionStatsReports = [];
        const analyticsSession = this._localPerson._analytics;
        if (analyticsSession) {
            // AnalyticsServerに初回接続できなかった場合のタイマー再セット処理
            analyticsSession.onConnectionStateChanged.add((state) => {
                if (state === 'connected' &&
                    this._waitingSendSubscriptionStatsReports.length > 0) {
                    for (const producerId of this._waitingSendSubscriptionStatsReports) {
                        if (this._producer && this._producer.id === producerId) {
                            this.startSendSubscriptionStatsReportTimer();
                        }
                    }
                    this._waitingSendSubscriptionStatsReports = [];
                }
            });
        }
    }
    _setConnectionState(state) {
        if (this._connectionState === state) {
            return;
        }
        log.debug('_setConnectionState', {
            state,
            forwardingId: this.forwardingId,
        });
        this._connectionState = state;
        this.onConnectionStateChanged.emit(state);
    }
    toJSON() {
        return {
            forwarding: this.forwarding,
            broadcasterTransport: this._broadcasterTransport,
            _connectionState: this._connectionState,
        };
    }
    /**@throws {SkyWayError} */
    startForwarding(configure) {
        return __awaiter(this, void 0, void 0, function* () {
            if (this.publication.contentType === 'data') {
                throw (0, core_1.createError)({
                    operationName: 'Sender.startForwarding',
                    context: this._context,
                    info: errors_1.errors.dataStreamNotSupported,
                    path: log.prefix,
                    channel: this.channel,
                });
            }
            const stream = this.publication.stream;
            if (!stream) {
                throw (0, core_1.createError)({
                    operationName: 'Sender.startForwarding',
                    context: this._context,
                    info: errors_1.errors.streamNotExistInPublication,
                    path: log.prefix,
                    channel: this.channel,
                });
            }
            this.onConnectionStateChanged
                .add((state) => {
                var _a;
                log.debug('transport connection state changed', (_a = this._broadcasterTransport) === null || _a === void 0 ? void 0 : _a.id, state);
                stream._setConnectionState(this._bot, state);
            })
                .disposer(this._disposer);
            log.debug('[start] Sender startForwarding', {
                botId: this._bot.id,
                publicationId: this.publication.id,
                contentType: this.publication.contentType,
                maxSubscribers: configure.maxSubscribers,
            });
            const { forwardingId, broadcasterTransportId, 
            // optional
            broadcasterTransportOptions, rtpCapabilities, identifierKey, } = yield this._api.startForwarding({
                botId: this._bot.id,
                publicationId: this.publication.id,
                contentType: this.publication.contentType,
                maxSubscribers: configure.maxSubscribers,
                publisherId: this.publication.publisher.id,
            });
            this.forwardingId = forwardingId;
            if (broadcasterTransportOptions) {
                log.debug('sender create new transport', {
                    broadcasterTransportOptions,
                });
                yield this._transportRepository.loadDevice(rtpCapabilities);
                this._broadcasterTransport = this._transportRepository.createTransport(this._localPerson.id, this._bot, broadcasterTransportOptions, 'send', this._iceManager, this._localPerson._analytics);
            }
            this._broadcasterTransport = this._transportRepository.getTransport(this._localPerson.id, broadcasterTransportId);
            if (!this._broadcasterTransport) {
                throw (0, core_1.createError)({
                    operationName: 'Sender.startForwarding',
                    context: this._context,
                    info: Object.assign(Object.assign({}, errors_1.errors.internal), { detail: '_broadcasterTransport not found' }),
                    path: log.prefix,
                    channel: this.channel,
                    payload: { broadcasterTransportOptions },
                });
            }
            this._broadcasterTransport.onConnectionStateChanged
                .add((state) => {
                this._setConnectionState(state);
            })
                .disposer(this._disposer);
            this._setConnectionState(this._broadcasterTransport.connectionState);
            const producer = yield this._produce(stream, this._broadcasterTransport);
            this._cleanupStreamCallbacks = this._setupTransportAccessForStream(stream, this._broadcasterTransport, producer);
            const analyticsSession = this._localPerson._analytics;
            if (analyticsSession && !analyticsSession.isClosed()) {
                if (analyticsSession.client.isConnectionEstablished()) {
                    this.startSendSubscriptionStatsReportTimer();
                }
                else {
                    // AnalyticsServerに初回接続できなかった場合はキューに入れる
                    this._waitingSendSubscriptionStatsReports.push(producer.id);
                }
            }
            log.debug('[end] Sender startForwarding', {
                forwardingId,
            });
            let relayingPublication = this.channel._getPublication(forwardingId);
            if (!relayingPublication) {
                relayingPublication = (yield this.channel.onStreamPublished
                    .watch((e) => e.publication.id === forwardingId, this._context.config.rtcApi.timeout)
                    .catch(() => {
                    throw (0, core_1.createError)({
                        operationName: 'Sender.startForwarding',
                        context: this._context,
                        info: Object.assign(Object.assign({}, errors_1.errors.timeout), { detail: 'SfuBotMember onStreamPublished' }),
                        path: log.prefix,
                        channel: this.channel,
                        payload: { forwardingId },
                    });
                })).publication;
            }
            const forwarding = new forwarding_1.Forwarding({
                configure,
                originPublication: this.publication,
                relayingPublication,
                api: this._api,
                context: this._context,
                identifierKey,
            });
            this.forwarding = forwarding;
            const botSubscribing = this.channel.subscriptions.find((s) => s.publication.id === this.publication.id);
            const [codec] = producer.rtpParameters.codecs;
            botSubscribing.codec = codec;
            if (this._localPerson._analytics &&
                this._localPerson._analytics.client.connectionState !== 'closed') {
                // 再送時に他の処理をブロックしないためにawaitしない
                void this._localPerson._analytics.client.sendBindingRtcPeerConnectionToSubscription({
                    subscriptionId: botSubscribing.id,
                    role: 'sender',
                    rtcPeerConnectionId: this._broadcasterTransport.id,
                });
            }
            if ((0, core_1.isSafari)()) {
                (0, core_1.waitForLocalStats)({
                    stream,
                    remoteMember: this._bot.id,
                    end: (stats) => {
                        const outbound = stats.find((s) => s.id.includes('RTCOutboundRTP') || s.type.includes('outbound-rtp'));
                        if ((outbound === null || outbound === void 0 ? void 0 : outbound.keyFramesEncoded) > 0)
                            return true;
                        return false;
                    },
                    interval: 10,
                })
                    .then(() => __awaiter(this, void 0, void 0, function* () {
                    const encodings = this.publication.encodings;
                    if ((encodings === null || encodings === void 0 ? void 0 : encodings.length) > 0) {
                        yield (0, core_1.setEncodingParams)(producer.rtpSender, encodings).catch((e) => {
                            log.error('_onEncodingsChanged failed', e, this);
                        });
                    }
                }))
                    .catch((err) => {
                    log.error('setEncodingParams waitForLocalStats failed', err, this);
                });
            }
            (0, core_1.waitForLocalStats)({
                stream,
                remoteMember: this._bot.id,
                end: (stats) => !!stats.find((s) => s.type.includes('local-candidate')),
            })
                .then(() => __awaiter(this, void 0, void 0, function* () {
                const payload = yield (0, core_1.createLogPayload)({
                    operationName: 'startForwarding/waitForLocalStats',
                    channel: this.channel,
                });
                log.debug(payload, 'forwarding connection connected', {
                    broadcasterTransportId,
                });
            }))
                .catch(() => { });
            return forwarding;
        });
    }
    _listenStreamEnableChange(stream) {
        if (this._unsubscribeStreamEnableChange) {
            this._unsubscribeStreamEnableChange();
        }
        const { removeListener } = stream._onEnableChanged.add((track) => __awaiter(this, void 0, void 0, function* () {
            yield this._replaceTrack(track).catch((e) => {
                log.warn((0, util_1.createWarnPayload)({
                    detail: 'replaceTrack failed',
                    operationName: 'Sender._listenStreamEnableChange',
                    bot: this._bot,
                    payload: e,
                }));
            });
        }));
        this._unsubscribeStreamEnableChange = removeListener;
    }
    _produce(stream, transport) {
        var _a, _b, _c, _d, _e, _f, _g, _h;
        return __awaiter(this, void 0, void 0, function* () {
            this.publication._onReplaceStream
                .add(({ newStream }) => __awaiter(this, void 0, void 0, function* () {
                if (!this._broadcasterTransport) {
                    throw (0, core_1.createError)({
                        operationName: 'Sender._produce',
                        context: this._context,
                        info: Object.assign(Object.assign({}, errors_1.errors.internal), { detail: '_broadcasterTransport not found' }),
                        path: log.prefix,
                        channel: this.channel,
                    });
                }
                this._listenStreamEnableChange(newStream);
                if (this._cleanupStreamCallbacks) {
                    this._cleanupStreamCallbacks();
                }
                this._cleanupStreamCallbacks = this._setupTransportAccessForStream(newStream, this._broadcasterTransport, producer);
                yield this._replaceTrack(newStream.track);
            }))
                .disposer(this._disposer);
            this._listenStreamEnableChange(stream);
            const transactionId = (0, core_1.uuidV4)();
            const producerOptions = {
                track: stream.track,
                // mediasoup-clientはデフォルトでunproduce時にtrack.stopを実行する
                stopTracks: false,
                appData: { transactionId },
                // デフォルトで一度mutedなTrackをProduceするとreplaceTrackしたTrackがDisableされる
                disableTrackOnPause: false,
            };
            const encodings = this.publication.encodings;
            if (encodings) {
                producerOptions.encodings = encodings;
            }
            this.publication._onEncodingsChanged
                .add((encodings) => __awaiter(this, void 0, void 0, function* () {
                yield (0, core_1.setEncodingParams)(producer.rtpSender, encodings).catch((e) => {
                    log.error('_onEncodingsChanged failed', e, this);
                });
            }))
                .disposer(this._disposer);
            const codecCapabilities = this.publication.codecCapabilities;
            const deviceCodecs = (_b = (_a = this._transportRepository.rtpCapabilities) === null || _a === void 0 ? void 0 : _a.codecs) !== null && _b !== void 0 ? _b : [];
            log.debug('select codec', { codecCapabilities, deviceCodecs });
            const [codec] = codecCapabilities.map((cap) => {
                if (cap.mimeType.toLowerCase().includes('video')) {
                    const codec = deviceCodecs.find((c) => {
                        var _a;
                        if (c.mimeType.toLowerCase() !== cap.mimeType.toLowerCase()) {
                            return false;
                        }
                        if (Object.keys((_a = cap.parameters) !== null && _a !== void 0 ? _a : {}).length > 0 &&
                            !(0, isEqual_1.default)(cap.parameters, c.parameters)) {
                            return false;
                        }
                        return true;
                    });
                    return codec;
                }
                const codec = deviceCodecs.find((c) => c.mimeType.toLowerCase() === cap.mimeType.toLowerCase());
                return codec;
            });
            log.debug('selected codec', { codec });
            if (codec) {
                const [codecType, codecName] = codec.mimeType.split('/');
                producerOptions.codec = Object.assign(Object.assign({}, codec), { mimeType: `${codecType}/${codecName.toUpperCase()}` });
                if (stream.contentType === 'video') {
                    this._fixVideoCodecWithParametersOrder(codec);
                }
            }
            else if (codecCapabilities.length > 0) {
                log.warn('preferred codec not supported', (0, util_1.createWarnPayload)({
                    channel: this.channel,
                    detail: 'preferred codec not supported',
                    operationName: 'Sender._produce',
                    bot: this._bot,
                    payload: {
                        codecCapabilities,
                        deviceCodecs,
                    },
                }));
            }
            if (stream.contentType === 'audio') {
                // apply opusDtx
                const opusDtx = (_d = (_c = codecCapabilities.find((c) => c.mimeType.toLowerCase() === 'audio/opus')) === null || _c === void 0 ? void 0 : _c.parameters) === null || _d === void 0 ? void 0 : _d.usedtx;
                if (opusDtx !== false) {
                    producerOptions.codecOptions = Object.assign(Object.assign({}, producerOptions.codecOptions), { opusDtx: true });
                }
                // apply opusStereo
                const opusStereo = (_f = (_e = codecCapabilities.find((c) => c.mimeType.toLowerCase() === 'audio/opus')) === null || _e === void 0 ? void 0 : _e.parameters) === null || _f === void 0 ? void 0 : _f.stereo;
                if (opusStereo) {
                    producerOptions.codecOptions = Object.assign(Object.assign({}, producerOptions.codecOptions), { opusStereo: true });
                }
                // apply opusFec
                const opusFec = (_h = (_g = codecCapabilities.find((c) => c.mimeType.toLowerCase() === 'audio/opus')) === null || _g === void 0 ? void 0 : _g.parameters) === null || _h === void 0 ? void 0 : _h.useinbandfec;
                if (opusFec) {
                    producerOptions.codecOptions = Object.assign(Object.assign({}, producerOptions.codecOptions), { opusFec: true });
                }
            }
            transport.onProduce
                .watch((p) => { var _a; return ((_a = p.producerOptions.appData) === null || _a === void 0 ? void 0 : _a.transactionId) === transactionId; }, this._context.config.rtcConfig.timeout)
                .then((producer) => __awaiter(this, void 0, void 0, function* () {
                try {
                    const { producerId } = yield this._api.createProducer({
                        botId: this._bot.id,
                        transportId: transport.id,
                        forwardingId: this.forwardingId,
                        producerOptions: producer.producerOptions,
                    });
                    producer.callback({ id: producerId });
                }
                catch (error) {
                    producer.errback(error);
                }
            }))
                .catch((e) => {
                log.error('onProduce failed', e, this);
            });
            log.debug('[start] msTransport.produce', this);
            const producer = yield transport.msTransport
                .produce(producerOptions)
                .catch((err) => {
                throw (0, core_1.createError)({
                    operationName: 'Sender._produce',
                    context: this._context,
                    info: Object.assign(Object.assign({}, errors_1.errors.internal), { detail: 'msTransport.produce failed' }),
                    path: log.prefix,
                    channel: this.channel,
                    error: err,
                });
            });
            log.debug('[end] msTransport.produce', this);
            this._producer = producer;
            return producer;
        });
    }
    /** @description 引数のParametersを持ったCodecを優先度配列の先頭に持ってくる
     *  @description H264対応のため
     */
    _fixVideoCodecWithParametersOrder(codec) {
        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
        //@ts-ignore
        const handler = this._broadcasterTransport.msTransport._handler;
        const findCodecWithParameters = (c) => {
            if (c.mimeType === codec.mimeType) {
                if (codec.parameters && Object.keys(codec.parameters).length > 0) {
                    if ((0, isEqual_1.default)(c.parameters, codec.parameters)) {
                        return true;
                    }
                    return false;
                }
                return true;
            }
            return false;
        };
        const copyCodecExceptPayloadType = (target, src) => {
            for (const key of Object.keys(target)) {
                if (key === 'payloadType') {
                    continue;
                }
                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
                //@ts-ignore
                target[key] = src[key];
            }
        };
        if (handler._sendingRtpParametersByKind) {
            const parameters = handler._sendingRtpParametersByKind['video'];
            const target = parameters.codecs.find(findCodecWithParameters);
            if (parameters && target) {
                const origin = JSON.parse(JSON.stringify(parameters));
                const [head] = parameters.codecs;
                const copyOfHead = JSON.parse(JSON.stringify(head));
                // 目的のRtpCodecParametersと先頭のRtpCodecParametersを入れ替える
                copyCodecExceptPayloadType(head, target);
                copyCodecExceptPayloadType(target, copyOfHead);
                log.debug('sort _sendingRtpParametersByKind', {
                    origin,
                    new: parameters.codecs,
                });
            }
        }
        if (handler._sendingRemoteRtpParametersByKind) {
            const parameters = handler._sendingRemoteRtpParametersByKind['video'];
            const target = parameters.codecs.find(findCodecWithParameters);
            if (parameters && target) {
                const origin = JSON.parse(JSON.stringify(parameters));
                const [head] = parameters.codecs;
                const copyOfHead = JSON.parse(JSON.stringify(head));
                // 目的のRtpCodecParametersと先頭のRtpCodecParametersを入れ替える
                copyCodecExceptPayloadType(head, target);
                copyCodecExceptPayloadType(target, copyOfHead);
                log.debug('sort _sendingRemoteRtpParametersByKind', {
                    origin,
                    new: parameters.codecs,
                });
            }
        }
    }
    _setupTransportAccessForStream(stream, transport, producer) {
        stream._getTransportCallbacks[this._bot.id] = () => ({
            rtcPeerConnection: transport.pc,
            connectionState: transport.connectionState,
            info: this,
        });
        stream._getStatsCallbacks[this._bot.id] = () => __awaiter(this, void 0, void 0, function* () {
            if (producer.closed) {
                delete stream._getStatsCallbacks[this._bot.id];
                return [];
            }
            const stats = yield producer.getStats();
            let arr = (0, core_1.statsToArray)(stats);
            arr = arr.map((stats) => {
                stats['sfuTransportId'] = transport.id;
                return stats;
            });
            return arr;
        });
        // replaceStream時に古いstreamに紐づくcallbackを削除するため、戻り値としてcallback削除用の関数を返し、replaceStream時に呼び出す
        const cleanupCallbacks = () => {
            delete stream._getTransportCallbacks[this._bot.id];
            delete stream._getStatsCallbacks[this._bot.id];
        };
        this._disposer.push(() => {
            cleanupCallbacks();
        });
        return cleanupCallbacks;
    }
    unproduce() {
        if (!this._producer) {
            return;
        }
        this._producer.close();
        this._producer = undefined;
        if (this.sendSubscriptionStatsReportTimer) {
            clearInterval(this.sendSubscriptionStatsReportTimer);
        }
    }
    _replaceTrack(track) {
        var _a, _b;
        return __awaiter(this, void 0, void 0, function* () {
            yield ((_b = (_a = this._producer) === null || _a === void 0 ? void 0 : _a.replaceTrack) === null || _b === void 0 ? void 0 : _b.call(_a, { track }).catch((e) => {
                throw (0, core_1.createError)({
                    operationName: 'Sender._replaceTrack',
                    context: this._context,
                    info: errors_1.errors.internal,
                    error: e,
                    path: log.prefix,
                    channel: this.channel,
                });
            }));
        });
    }
    close() {
        this.closed = true;
        if (this._unsubscribeStreamEnableChange) {
            this._unsubscribeStreamEnableChange();
        }
        this._setConnectionState('disconnected');
        this._disposer.dispose();
    }
    get pc() {
        var _a;
        return (_a = this._broadcasterTransport) === null || _a === void 0 ? void 0 : _a.pc;
    }
    startSendSubscriptionStatsReportTimer() {
        const analyticsSession = this._localPerson._analytics;
        const subscription = this._bot.subscriptions.find((s) => s.publication.id === this.publication.id);
        if (subscription && analyticsSession) {
            const intervalSec = analyticsSession.client.getIntervalSec();
            this.sendSubscriptionStatsReportTimer = setInterval(() => __awaiter(this, void 0, void 0, function* () {
                // AnalyticsSessionがcloseされていたらタイマーを止める
                if (!analyticsSession || analyticsSession.isClosed()) {
                    if (this.sendSubscriptionStatsReportTimer) {
                        clearInterval(this.sendSubscriptionStatsReportTimer);
                    }
                    return;
                }
                if (this._producer) {
                    const stats = yield this._producer.getStats();
                    if (stats) {
                        // 再送時に他の処理をブロックしないためにawaitしない
                        void analyticsSession.client.sendSubscriptionStatsReport(stats, {
                            subscriptionId: subscription.id,
                            role: 'sender',
                            contentType: this.publication.contentType,
                            createdAt: Date.now(),
                        });
                    }
                }
            }), intervalSec * 1000);
        }
    }
}
exports.Sender = Sender;
//# sourceMappingURL=sender.js.map