UNPKG

shaka-player

Version:
1,403 lines (1,216 loc) 96 kB
/** * @license * Copyright 2016 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ goog.provide('shaka.media.StreamingEngine'); goog.require('goog.asserts'); goog.require('shaka.log'); goog.require('shaka.media.MediaSourceEngine'); goog.require('shaka.net.Backoff'); goog.require('shaka.net.NetworkingEngine'); goog.require('shaka.util.DelayedTick'); goog.require('shaka.util.Error'); goog.require('shaka.util.FakeEvent'); goog.require('shaka.util.Functional'); goog.require('shaka.util.IDestroyable'); goog.require('shaka.util.Iterables'); goog.require('shaka.util.ManifestParserUtils'); goog.require('shaka.util.MimeUtils'); goog.require('shaka.util.Mp4Parser'); goog.require('shaka.util.Networking'); goog.require('shaka.util.Periods'); goog.require('shaka.util.PublicPromise'); /** * Creates a StreamingEngine. * * The StreamingEngine is responsible for setting up the Manifest's Streams * (i.e., for calling each Stream's createSegmentIndex() function), for * downloading segments, for co-ordinating audio, video, and text buffering, * and for handling Period transitions. The StreamingEngine provides an * interface to switch between Streams, but it does not choose which Streams to * switch to. * * The StreamingEngine notifies its owner when it needs to buffer a new Period, * so its owner can choose which Streams within that Period to initially * buffer. Moreover, the StreamingEngine also notifies its owner when any * Stream within the current Period may be switched to, so its owner can switch * bitrates, resolutions, or languages. * * The StreamingEngine does not need to be notified about changes to the * Manifest's SegmentIndexes; however, it does need to be notified when new * Periods are added to the Manifest, so it can set up that Period's Streams. * * To start the StreamingEngine the owner must first call configure() followed * by init(). The StreamingEngine will then call onChooseStreams(p) when it * needs to buffer Period p; it will then switch to the Streams returned from * that function. The StreamingEngine will call onCanSwitch() when any * Stream within the current Period may be switched to. * * The owner must call seeked() each time the playhead moves to a new location * within the presentation timeline; however, the owner may forego calling * seeked() when the playhead moves outside the presentation timeline. * * @param {shaka.extern.Manifest} manifest * @param {shaka.media.StreamingEngine.PlayerInterface} playerInterface * * @constructor * @struct * @implements {shaka.util.IDestroyable} */ shaka.media.StreamingEngine = function(manifest, playerInterface) { /** @private {?shaka.media.StreamingEngine.PlayerInterface} */ this.playerInterface_ = playerInterface; /** @private {?shaka.extern.Manifest} */ this.manifest_ = manifest; /** @private {?shaka.extern.StreamingConfiguration} */ this.config_ = null; /** @private {number} */ this.bufferingGoalScale_ = 1; /** @private {Promise} */ this.setupPeriodPromise_ = Promise.resolve(); /** * Maps a Period's index to an object that indicates that either * 1. the Period has not been set up (undefined). * 2. the Period is being set up ([a PublicPromise, false]). * 3. the Period is set up (i.e., all Streams within the Period are set up) * and can be switched to ([a PublicPromise, true]). * * @private {Array.<?{promise: shaka.util.PublicPromise, resolved: boolean}>} */ this.canSwitchPeriod_ = []; /** * Maps a Stream's ID to an object that indicates that either * 1. the Stream has not been set up (undefined). * 2. the Stream is being set up ([a Promise instance, false]). * 3. the Stream is set up and can be switched to * ([a Promise instance, true]). * * @private {!Map.<number, * ?{promise: shaka.util.PublicPromise, resolved: boolean}>} */ this.canSwitchStream_ = new Map(); /** * Maps a content type, e.g., 'audio', 'video', or 'text', to a MediaState. * * @private {!Map.<shaka.util.ManifestParserUtils.ContentType, !shaka.media.StreamingEngine.MediaState_>} */ this.mediaStates_ = new Map(); /** * Set to true once one segment of each content type has been buffered. * * @private {boolean} */ this.startupComplete_ = false; /** * Used for delay and backoff of failure callbacks, so that apps do not retry * instantly. * * @private {shaka.net.Backoff} */ this.failureCallbackBackoff_ = null; /** * Set to true on fatal error. Interrupts fetchAndAppend_(). * * @private {boolean} */ this.fatalError_ = false; /** @private {boolean} */ this.destroyed_ = false; /** * Set to true when a request to unload text stream comes in. This is used * since loading new text stream is async, the request of unloading text * stream might come in before setting up new text stream is finished. * @private {boolean} */ this.unloadingTextStream_ = false; /** @private {number} */ this.textStreamSequenceId_ = 0; }; /** * @typedef {{ * variant: (?shaka.extern.Variant|undefined), * text: ?shaka.extern.Stream * }} * * @property {(?shaka.extern.Variant|undefined)} variant * The chosen variant. May be omitted for text re-init. * @property {?shaka.extern.Stream} text * The chosen text stream. */ shaka.media.StreamingEngine.ChosenStreams; /** * @typedef {{ * getPresentationTime: function():number, * getBandwidthEstimate: function():number, * mediaSourceEngine: !shaka.media.MediaSourceEngine, * netEngine: shaka.net.NetworkingEngine, * onChooseStreams: function(!shaka.extern.Period): * shaka.media.StreamingEngine.ChosenStreams, * onCanSwitch: function(), * onError: function(!shaka.util.Error), * onEvent: function(!Event), * onManifestUpdate: function(), * onSegmentAppended: function(), * onInitialStreamsSetup: (function()|undefined), * onStartupComplete: (function()|undefined) * }} * * @property {function():number} getPresentationTime * Get the position in the presentation (in seconds) of the content that the * viewer is seeing on screen right now. * @property {function():number} getBandwidthEstimate * Get the estimated bandwidth in bits per second. * @property {!shaka.media.MediaSourceEngine} mediaSourceEngine * The MediaSourceEngine. The caller retains ownership. * @property {shaka.net.NetworkingEngine} netEngine * The NetworkingEngine instance to use. The caller retains ownership. * @property {function(!shaka.extern.Period): * shaka.media.StreamingEngine.ChosenStreams} onChooseStreams * Called by StreamingEngine when the given Period needs to be buffered. * StreamingEngine will switch to the variant and text stream returned from * this function. * The owner cannot call switch() directly until the StreamingEngine calls * onCanSwitch(). * @property {function()} onCanSwitch * Called by StreamingEngine when the Period is set up and switching is * permitted. * @property {function(!shaka.util.Error)} onError * Called when an error occurs. If the error is recoverable (see * {@link shaka.util.Error}) then the caller may invoke either * StreamingEngine.switch*() or StreamingEngine.seeked() to attempt recovery. * @property {function(!Event)} onEvent * Called when an event occurs that should be sent to the app. * @property {function()} onManifestUpdate * Called when an embedded 'emsg' box should trigger a manifest update. * @property {function()} onSegmentAppended * Called after a segment is successfully appended to a MediaSource. * @property {(function()|undefined)} onInitialStreamsSetup * Optional callback which is called when the initial set of Streams have been * setup. Intended to be used by tests. * @property {(function()|undefined)} onStartupComplete * Optional callback which is called when startup has completed. Intended to * be used by tests. */ shaka.media.StreamingEngine.PlayerInterface; /** * @typedef {{ * type: shaka.util.ManifestParserUtils.ContentType, * stream: shaka.extern.Stream, * lastStream: ?shaka.extern.Stream, * lastSegmentReference: shaka.media.SegmentReference, * restoreStreamAfterTrickPlay: ?shaka.extern.Stream, * needInitSegment: boolean, * needPeriodIndex: number, * endOfStream: boolean, * performingUpdate: boolean, * updateTimer: shaka.util.DelayedTick, * waitingToClearBuffer: boolean, * waitingToFlushBuffer: boolean, * clearBufferSafeMargin: number, * clearingBuffer: boolean, * recovering: boolean, * hasError: boolean, * resumeAt: number, * operation: shaka.net.NetworkingEngine.PendingRequest * }} * * @description * Contains the state of a logical stream, i.e., a sequence of segmented data * for a particular content type. At any given time there is a Stream object * associated with the state of the logical stream. * * @property {shaka.util.ManifestParserUtils.ContentType} type * The stream's content type, e.g., 'audio', 'video', or 'text'. * @property {shaka.extern.Stream} stream * The current Stream. * @property {?shaka.extern.Stream} lastStream * The Stream of the last segment that was appended. * @property {shaka.media.SegmentReference} lastSegmentReference * The SegmentReference of the last segment that was appended. * @property {?shaka.extern.Stream} restoreStreamAfterTrickPlay * The Stream to restore after trick play mode is turned off. * @property {boolean} needInitSegment * True indicates that |stream|'s init segment must be inserted before the * next media segment is appended. * @property {boolean} endOfStream * True indicates that the end of the buffer has hit the end of the * presentation. * @property {number} needPeriodIndex * The index of the Period which needs to be buffered. * @property {boolean} performingUpdate * True indicates that an update is in progress. * @property {shaka.util.DelayedTick} updateTimer * A timer used to update the media state. * @property {boolean} waitingToClearBuffer * True indicates that the buffer must be cleared after the current update * finishes. * @property {boolean} waitingToFlushBuffer * True indicates that the buffer must be flushed after it is cleared. * @property {number} clearBufferSafeMargin * The amount of buffer to retain when clearing the buffer after the update. * @property {boolean} clearingBuffer * True indicates that the buffer is being cleared. * @property {boolean} recovering * True indicates that the last segment was not appended because it could not * fit in the buffer. * @property {boolean} hasError * True indicates that the stream has encountered an error and has stopped * updating. * @property {number} resumeAt * An override for the time to start performing updates at. If the playhead * is behind this time, update_() will still start fetching segments from * this time. If the playhead is ahead of the time, this field is ignored. * @property {shaka.net.NetworkingEngine.PendingRequest} operation * Operation with the number of bytes to be downloaded. */ shaka.media.StreamingEngine.MediaState_; /** * The fudge factor for appendWindowStart. By adjusting the window backward, we * avoid rounding errors that could cause us to remove the keyframe at the start * of the Period. * * NOTE: This was increased as part of the solution to * https://github.com/google/shaka-player/issues/1281 * * @const {number} * @private */ shaka.media.StreamingEngine.APPEND_WINDOW_START_FUDGE_ = 0.1; /** * The fudge factor for appendWindowEnd. By adjusting the window backward, we * avoid rounding errors that could cause us to remove the last few samples of * the Period. This rounding error could then create an artificial gap and a * stutter when the gap-jumping logic takes over. * * https://github.com/google/shaka-player/issues/1597 * * @const {number} * @private */ shaka.media.StreamingEngine.APPEND_WINDOW_END_FUDGE_ = 0.01; /** * The maximum number of segments by which a stream can get ahead of other * streams. * * Introduced to keep StreamingEngine from letting one media type get too far * ahead of another. For example, audio segments are typically much smaller * than video segments, so in the time it takes to fetch one video segment, we * could fetch many audio segments. This doesn't help with buffering, though, * since the intersection of the two buffered ranges is what counts. * * @const {number} * @private */ shaka.media.StreamingEngine.MAX_RUN_AHEAD_SEGMENTS_ = 1; /** @override */ shaka.media.StreamingEngine.prototype.destroy = async function() { const aborts = []; for (const state of this.mediaStates_.values()) { this.cancelUpdate_(state); aborts.push(this.abortOperations_(state)); } await Promise.all(aborts); this.mediaStates_.clear(); this.canSwitchStream_.clear(); this.playerInterface_ = null; this.manifest_ = null; this.setupPeriodPromise_ = null; this.canSwitchPeriod_ = null; this.config_ = null; this.destroyed_ = true; }; /** * Called by the Player to provide an updated configuration any time it changes. * Must be called at least once before init(). * * @param {shaka.extern.StreamingConfiguration} config */ shaka.media.StreamingEngine.prototype.configure = function(config) { this.config_ = config; // Create separate parameters for backoff during streaming failure. /** @type {shaka.extern.RetryParameters} */ let failureRetryParams = { // The term "attempts" includes the initial attempt, plus all retries. // In order to see a delay, there would have to be at least 2 attempts. maxAttempts: Math.max(config.retryParameters.maxAttempts, 2), baseDelay: config.retryParameters.baseDelay, backoffFactor: config.retryParameters.backoffFactor, fuzzFactor: config.retryParameters.fuzzFactor, timeout: 0, // irrelevant }; // We don't want to ever run out of attempts. The application should be // allowed to retry streaming infinitely if it wishes. let autoReset = true; this.failureCallbackBackoff_ = new shaka.net.Backoff(failureRetryParams, autoReset); }; /** * Initialize and start streaming. * * By calling this method, streaming engine will choose the initial streams by * calling out to |onChooseStreams| followed by |onCanSwitch|. When streaming * engine switches periods, it will call |onChooseStreams| followed by * |onCanSwitch|. * * Asking streaming engine to switch streams between |onChooseStreams| and * |onChangeSwitch| is not supported. * * After the StreamingEngine calls onChooseStreams(p) for the first time, it * will begin setting up the Streams returned from that function and * subsequently switch to them. However, the StreamingEngine will not begin * setting up any other Streams until at least one segment from each of the * initial set of Streams has been buffered (this reduces startup latency). * * After the StreamingEngine completes this startup phase it will begin setting * up each Period's Streams (while buffering in parrallel). * * When the StreamingEngine needs to buffer the next Period it will have * already set up that Period's Streams. So, when the StreamingEngine calls * onChooseStreams(p) after the first time, the StreamingEngine will * immediately switch to the Streams returned from that function. * * @return {!Promise} */ shaka.media.StreamingEngine.prototype.start = async function() { goog.asserts.assert(this.config_, 'StreamingEngine configure() must be called before init()!'); // Determine which Period we must buffer. const presentationTime = this.playerInterface_.getPresentationTime(); const needPeriodIndex = this.findPeriodForTime_(presentationTime); // Get the initial set of Streams. const initialStreams = this.playerInterface_.onChooseStreams( this.manifest_.periods[needPeriodIndex]); if (!initialStreams.variant && !initialStreams.text) { shaka.log.error('init: no Streams chosen'); return new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.STREAMING, shaka.util.Error.Code.INVALID_STREAMS_CHOSEN); } // Setup the initial set of Streams and then begin each update cycle. After // startup completes onUpdate_() will set up the remaining Periods. await this.initStreams_( initialStreams.variant ? initialStreams.variant.audio : null, initialStreams.variant ? initialStreams.variant.video : null, initialStreams.text, presentationTime); if (this.destroyed_) { return; } shaka.log.debug('init: completed initial Stream setup'); // Subtlety: onInitialStreamsSetup() may call switch() or seeked(), so we // must schedule an update beforehand so |updateTimer| is set. if (this.playerInterface_ && this.playerInterface_.onInitialStreamsSetup) { shaka.log.v1('init: calling onInitialStreamsSetup()...'); this.playerInterface_.onInitialStreamsSetup(); } }; /** * Gets the Period in which we are currently buffering. This might be different * from the Period which contains the Playhead. * @return {?shaka.extern.Period} */ shaka.media.StreamingEngine.prototype.getBufferingPeriod = function() { const ContentType = shaka.util.ManifestParserUtils.ContentType; const video = this.mediaStates_.get(ContentType.VIDEO); if (video) { return this.manifest_.periods[video.needPeriodIndex]; } const audio = this.mediaStates_.get(ContentType.AUDIO); if (audio) { return this.manifest_.periods[audio.needPeriodIndex]; } return null; }; /** * Get the audio stream which we are currently buffering. Returns null if there * is no audio streaming. * @return {?shaka.extern.Stream} */ shaka.media.StreamingEngine.prototype.getBufferingAudio = function() { const ContentType = shaka.util.ManifestParserUtils.ContentType; return this.getStream_(ContentType.AUDIO); }; /** * Get the video stream which we are currently buffering. Returns null if there * is no video streaming. * @return {?shaka.extern.Stream} */ shaka.media.StreamingEngine.prototype.getBufferingVideo = function() { const ContentType = shaka.util.ManifestParserUtils.ContentType; return this.getStream_(ContentType.VIDEO); }; /** * Get the text stream which we are currently buffering. Returns null if there * is no text streaming. * @return {?shaka.extern.Stream} */ shaka.media.StreamingEngine.prototype.getBufferingText = function() { const ContentType = shaka.util.ManifestParserUtils.ContentType; return this.getStream_(ContentType.TEXT); }; /** * Get the stream of the given type which we are currently buffering. Returns * null if there is no stream for the given type. * @param {shaka.util.ManifestParserUtils.ContentType} type * @return {?shaka.extern.Stream} * @private */ shaka.media.StreamingEngine.prototype.getStream_ = function(type) { const state = this.mediaStates_.get(type); if (state) { // Don't tell the caller about trick play streams. If we're in trick // play, return the stream we will go back to after we exit trick play. return state.restoreStreamAfterTrickPlay || state.stream; } else { return null; } }; /** * Notifies StreamingEngine that a new text stream was added to the manifest. * This initializes the given stream. This returns a Promise that resolves when * the stream has been set up, and a media state has been created. * * @param {shaka.extern.Stream} stream * @return {!Promise} */ shaka.media.StreamingEngine.prototype.loadNewTextStream = async function( stream) { const ContentType = shaka.util.ManifestParserUtils.ContentType; // Clear MediaSource's buffered text, so that the new text stream will // properly replace the old buffered text. await this.playerInterface_.mediaSourceEngine.clear(ContentType.TEXT); // Since setupStreams_() is async, if the user hides/shows captions quickly, // there would be a race condition that a new text media state is created // but the old media state is not yet deleted. // The Sequence Id is to avoid that race condition. this.textStreamSequenceId_++; this.unloadingTextStream_ = false; let currentSequenceId = this.textStreamSequenceId_; let mediaSourceEngine = this.playerInterface_.mediaSourceEngine; const streamMap = new Map(); const streamSet = new Set(); streamMap.set(ContentType.TEXT, stream); streamSet.add(stream); await mediaSourceEngine.init(streamMap, /** forceTansmuxTS */ false); if (this.destroyed_) { return; } await this.setupStreams_(streamSet); if (this.destroyed_) { return; } const showText = this.playerInterface_ .mediaSourceEngine .getTextDisplayer() .isTextVisible(); const streamText = showText || this.config_.alwaysStreamText; if ((this.textStreamSequenceId_ == currentSequenceId) && !this.mediaStates_.has(ContentType.TEXT) && !this.unloadingTextStream_ && streamText) { const presentationTime = this.playerInterface_.getPresentationTime(); const needPeriodIndex = this.findPeriodForTime_(presentationTime); const state = this.createMediaState_(stream, needPeriodIndex, /* resumeAt= */ 0); this.mediaStates_.set(ContentType.TEXT, state); this.scheduleUpdate_(state, 0); } }; /** * Stop fetching text stream when the user chooses to hide the captions. */ shaka.media.StreamingEngine.prototype.unloadTextStream = function() { const ContentType = shaka.util.ManifestParserUtils.ContentType; this.unloadingTextStream_ = true; const state = this.mediaStates_.get(ContentType.TEXT); if (state) { this.cancelUpdate_(state); this.abortOperations_(state).catch(() => {}); this.mediaStates_.delete(ContentType.TEXT); } }; /** * Set trick play on or off. * If trick play is on, related trick play streams will be used when possible. * @param {boolean} on */ shaka.media.StreamingEngine.prototype.setTrickPlay = function(on) { const ContentType = shaka.util.ManifestParserUtils.ContentType; const mediaState = this.mediaStates_.get(ContentType.VIDEO); if (!mediaState) return; let stream = mediaState.stream; if (!stream) return; shaka.log.debug('setTrickPlay', on); if (on) { let trickModeVideo = stream.trickModeVideo; if (!trickModeVideo) return; // Can't engage trick play. let normalVideo = mediaState.restoreStreamAfterTrickPlay; if (normalVideo) return; // Already in trick play. shaka.log.debug('Engaging trick mode stream', trickModeVideo); this.switchInternal_(trickModeVideo, /* clearBuffer= */ false, /* safeMargin= */ 0, /* force= */ false); mediaState.restoreStreamAfterTrickPlay = stream; } else { let normalVideo = mediaState.restoreStreamAfterTrickPlay; if (!normalVideo) return; shaka.log.debug('Restoring non-trick-mode stream', normalVideo); mediaState.restoreStreamAfterTrickPlay = null; this.switchInternal_(normalVideo, /* clearBuffer= */ true, /* safeMargin= */ 0, /* force= */ false); } }; /** * @param {shaka.extern.Variant} variant * @param {boolean} clearBuffer * @param {number} safeMargin * @return {boolean} Whether we actually switched streams. */ shaka.media.StreamingEngine.prototype.switchVariant = function(variant, clearBuffer, safeMargin) { let ret = false; if (variant.video) { const changed = this.switchInternal_( variant.video, /* clearBuffer= */ clearBuffer, /* safeMargin= */ safeMargin, /* force= */ false); ret = ret || changed; } if (variant.audio) { const changed = this.switchInternal_( variant.audio, /* clearBuffer= */ clearBuffer, /* safeMargin= */ safeMargin, /* force= */ false); ret = ret || changed; } return ret; }; /** * @param {shaka.extern.Stream} textStream * @return {boolean} Whether we actually switched streams. */ shaka.media.StreamingEngine.prototype.switchTextStream = function(textStream) { const ContentType = shaka.util.ManifestParserUtils.ContentType; goog.asserts.assert(textStream && textStream.type == ContentType.TEXT, 'Wrong stream type passed to switchTextStream!'); return this.switchInternal_( textStream, /* clearBuffer= */ true, /* safeMargin= */ 0, /* force= */ false); }; /** Reload the current text stream. */ shaka.media.StreamingEngine.prototype.reloadTextStream = function() { const ContentType = shaka.util.ManifestParserUtils.ContentType; const mediaState = this.mediaStates_.get(ContentType.TEXT); if (mediaState) { // Don't reload if there's no text to begin with. this.switchInternal_(mediaState.stream, /* clearBuffer= */ true, /* safeMargin= */ 0, /* force= */ true); } }; /** * Switches to the given Stream. |stream| may be from any Variant or any Period. * * @param {shaka.extern.Stream} stream * @param {boolean} clearBuffer * @param {number} safeMargin * @param {boolean} force * If true, reload the text stream even if it did not change. * @return {boolean} * @private */ shaka.media.StreamingEngine.prototype.switchInternal_ = function( stream, clearBuffer, safeMargin, force) { const ContentType = shaka.util.ManifestParserUtils.ContentType; const type = /** @type {!ContentType} */(stream.type); const mediaState = this.mediaStates_.get(type); if (!mediaState && stream.type == ContentType.TEXT && this.config_.ignoreTextStreamFailures) { this.loadNewTextStream(stream); return true; } goog.asserts.assert(mediaState, 'switch: expected mediaState to exist'); if (!mediaState) { return false; } // If we are selecting a stream from a different Period, then we need to // handle a Period transition. Simply ignore the given stream, assuming that // Player will select the same track in onChooseStreams. let periodIndex = this.findPeriodContainingStream_(stream); const mediaStates = Array.from(this.mediaStates_.values()); const needSamePeriod = mediaStates.every((ms) => { return ms.needPeriodIndex == mediaState.needPeriodIndex; }); if (clearBuffer && periodIndex != mediaState.needPeriodIndex && needSamePeriod) { shaka.log.debug('switch: switching to stream in another Period; clearing ' + 'buffer and changing Periods'); // handlePeriodTransition_ will be called on the next update because the // current Period won't match the playhead Period. this.mediaStates_.forEach((mediaState) => { this.forceClearBuffer_(mediaState); }); return true; } if (mediaState.restoreStreamAfterTrickPlay) { shaka.log.debug('switch during trick play mode', stream); // Already in trick play mode, so stick with trick mode tracks if possible. if (stream.trickModeVideo) { // Use the trick mode stream, but revert to the new selection later. mediaState.restoreStreamAfterTrickPlay = stream; stream = stream.trickModeVideo; shaka.log.debug('switch found trick play stream', stream); } else { // There is no special trick mode video for this stream! mediaState.restoreStreamAfterTrickPlay = null; shaka.log.debug('switch found no special trick play stream'); } } // Ensure the Period is ready. let canSwitchRecord = this.canSwitchPeriod_[periodIndex]; goog.asserts.assert( canSwitchRecord && canSwitchRecord.resolved, 'switch: expected Period ' + periodIndex + ' to be ready'); if (!canSwitchRecord || !canSwitchRecord.resolved) { return false; } // Sanity check. If the Period is ready then the Stream should be ready too. canSwitchRecord = this.canSwitchStream_.get(stream.id); goog.asserts.assert(canSwitchRecord && canSwitchRecord.resolved, 'switch: expected Stream ' + stream.id + ' to be ready'); if (!canSwitchRecord || !canSwitchRecord.resolved) { return false; } if (mediaState.stream == stream && !force) { const streamTag = shaka.media.StreamingEngine.logPrefix_(mediaState); shaka.log.debug('switch: Stream ' + streamTag + ' already active'); return false; } if (stream.type == ContentType.TEXT) { // Mime types are allowed to change for text streams. // Reinitialize the text parser, but only if we are going to fetch the init // segment again. let fullMimeType = shaka.util.MimeUtils.getFullType( stream.mimeType, stream.codecs); this.playerInterface_.mediaSourceEngine.reinitText(fullMimeType); } mediaState.stream = stream; mediaState.needInitSegment = true; let streamTag = shaka.media.StreamingEngine.logPrefix_(mediaState); shaka.log.debug('switch: switching to Stream ' + streamTag); if (this.shouldAbortCurrentRequest_(mediaState, periodIndex)) { shaka.log.info('Aborting current segment request to switch.'); mediaState.operation.abort(); } if (clearBuffer) { if (mediaState.clearingBuffer) { // We are already going to clear the buffer, but make sure it is also // flushed. mediaState.waitingToFlushBuffer = true; } else if (mediaState.performingUpdate) { // We are performing an update, so we have to wait until it's finished. // onUpdate_() will call clearBuffer_() when the update has finished. // We need to save the safe margin because its value will be needed when // clearing the buffer after the update. mediaState.waitingToClearBuffer = true; mediaState.clearBufferSafeMargin = safeMargin; mediaState.waitingToFlushBuffer = true; } else { // Cancel the update timer, if any. this.cancelUpdate_(mediaState); // Clear right away. this.clearBuffer_(mediaState, /* flush */ true, safeMargin) .catch((error) => { if (this.playerInterface_) { this.playerInterface_.onError( /** @type {!shaka.util.Error} */ (error)); } }); } } return true; }; /** * Returns whether we should abort the current request. * * @param {!shaka.media.StreamingEngine.MediaState_} mediaState * @param {number} periodIndex * @return {boolean} */ shaka.media.StreamingEngine.prototype.shouldAbortCurrentRequest_ = function(mediaState, periodIndex) { // If the operation is completed, it will be set to null, and there's no need // to abort the request. if (!mediaState.operation) { return false; } const presentationTime = this.playerInterface_.getPresentationTime(); const bufferEnd = this.playerInterface_.mediaSourceEngine.bufferEnd(mediaState.type); // The next segment to append from the current stream. This doesn't account // for a pending network request and will likely be different from that since // we just switched. const newSegment = this.getSegmentReferenceNeeded_( mediaState, presentationTime, bufferEnd, periodIndex); let newSegmentSize = newSegment ? newSegment.getSize() : null; if (newSegment && !newSegmentSize) { // compute approximate segment size using stream bandwidth const duration = newSegment.getEndTime() - newSegment.getStartTime(); // bandwidth is in bits per second, and the size is in bytes newSegmentSize = duration * mediaState.stream.bandwidth / 8; } if (isNaN(newSegmentSize)) { return false; } // When switching, we'll need to download the init segment. const init = mediaState.stream.initSegmentReference; if (init) { newSegmentSize += init.getSize() || 0; } const bandwidthEstimate = this.playerInterface_.getBandwidthEstimate(); // The estimate is in bits per second, and the size is in bytes. The time // remaining is in seconds after this calculation. const timeToFetchNewSegment = (newSegmentSize * 8) / bandwidthEstimate; // If the new segment can be finished in time without risking a buffer // underflow, we should abort the old one and switch. const bufferedAhead = bufferEnd - presentationTime; const safetyBuffer = Math.max( this.manifest_.minBufferTime || 0, this.config_.rebufferingGoal); const safeBufferedAhead = bufferedAhead - safetyBuffer; if (timeToFetchNewSegment < safeBufferedAhead) { return true; } // If the thing we want to switch to will be done more quickly than what // we've got in progress, we should abort the old one and switch. const bytesRemaining = mediaState.operation.getBytesRemaining(); if (bytesRemaining > newSegmentSize) { return true; } // Otherwise, complete the operation in progress. return false; }; /** * Notifies the StreamingEngine that the playhead has moved to a valid time * within the presentation timeline. */ shaka.media.StreamingEngine.prototype.seeked = function() { const Iterables = shaka.util.Iterables; const presentationTime = this.playerInterface_.getPresentationTime(); const smallGapLimit = this.config_.smallGapLimit; const checkBuffered = (type) => { return this.playerInterface_.mediaSourceEngine.isBuffered( type, presentationTime, smallGapLimit); }; let streamCleared = false; const atPeriodIndex = this.findPeriodForTime_(presentationTime); const allSeekingWithinSamePeriod = Iterables.every( this.mediaStates_.values(), (state) => state.needPeriodIndex == atPeriodIndex); if (allSeekingWithinSamePeriod) { // If seeking to the same period you were in before, clear buffers // individually as desired. for (const type of this.mediaStates_.keys()) { if (!checkBuffered(type)) { // This stream exists, and isn't buffered. this.forceClearBuffer_(this.mediaStates_.get(type)); streamCleared = true; } } } else { // Only treat this as a buffered seek if every media state has a buffer. // For example, if we have buffered text but not video, we should still // clear every buffer so all media states need the same Period. const isAllBuffered = Iterables.every( this.mediaStates_.keys(), checkBuffered); if (!isAllBuffered) { // This was an unbuffered seek for at least one stream, so clear all // buffers. // Don't clear only some of the buffers because we can become stalled // since the media states are waiting for different Periods. shaka.log.debug('(all): seeked: unbuffered seek: clearing all buffers'); this.mediaStates_.forEach((mediaState) => { this.forceClearBuffer_(mediaState); }); streamCleared = true; } } if (!streamCleared) { shaka.log.debug( '(all): seeked: buffered seek: presentationTime=' + presentationTime); } }; /** * Clear the buffer for a given stream. Unlike clearBuffer_, this will handle * cases where a MediaState is performing an update. After this runs, every * MediaState will have a pending update. * @param {!shaka.media.StreamingEngine.MediaState_} mediaState * @private */ shaka.media.StreamingEngine.prototype.forceClearBuffer_ = function( mediaState) { const logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState); if (mediaState.clearingBuffer) { // We're already clearing the buffer, so we don't need to clear the // buffer again. shaka.log.debug(logPrefix, 'clear: already clearing the buffer'); return; } if (mediaState.waitingToClearBuffer) { // May not be performing an update, but an update will still happen. // See: https://github.com/google/shaka-player/issues/334 shaka.log.debug(logPrefix, 'clear: already waiting'); return; } if (mediaState.performingUpdate) { // We are performing an update, so we have to wait until it's finished. // onUpdate_() will call clearBuffer_() when the update has finished. shaka.log.debug(logPrefix, 'clear: currently updating'); mediaState.waitingToClearBuffer = true; // We can set the offset to zero to remember that this was a call to // clearAllBuffers. mediaState.clearBufferSafeMargin = 0; return; } const type = mediaState.type; if (this.playerInterface_.mediaSourceEngine.bufferStart(type) == null) { // Nothing buffered. shaka.log.debug(logPrefix, 'clear: nothing buffered'); if (mediaState.updateTimer == null) { // Note: an update cycle stops when we buffer to the end of the // presentation or Period, or when we raise an error. this.scheduleUpdate_(mediaState, 0); } return; } // An update may be scheduled, but we can just cancel it and clear the // buffer right away. Note: clearBuffer_() will schedule the next update. shaka.log.debug(logPrefix, 'clear: handling right now'); this.cancelUpdate_(mediaState); this.clearBuffer_(mediaState, /* flush */ false, 0).catch((error) => { if (this.playerInterface_) { this.playerInterface_.onError(/** @type {!shaka.util.Error} */ (error)); } }); }; /** * Initializes the given streams and media states if required. This will * schedule updates for the given types. * * @param {?shaka.extern.Stream} audio * @param {?shaka.extern.Stream} video * @param {?shaka.extern.Stream} text * @param {number} resumeAt * @return {!Promise} * @private */ shaka.media.StreamingEngine.prototype.initStreams_ = async function( audio, video, text, resumeAt) { goog.asserts.assert(this.config_, 'StreamingEngine configure() must be called before init()!'); // Determine which Period we must buffer. const presentationTime = this.playerInterface_.getPresentationTime(); const needPeriodIndex = this.findPeriodForTime_(presentationTime); // Init/re-init MediaSourceEngine. Note that a re-init is only valid for text. const ContentType = shaka.util.ManifestParserUtils.ContentType; /** * @type {!Map.<shaka.util.ManifestParserUtils.ContentType, * shaka.extern.Stream>} */ const streamsByType = new Map(); /** @type {!Set.<shaka.extern.Stream>} */ const streams = new Set(); if (audio) { streamsByType.set(ContentType.AUDIO, audio); streams.add(audio); } if (video) { streamsByType.set(ContentType.VIDEO, video); streams.add(video); } if (text) { streamsByType.set(ContentType.TEXT, text); streams.add(text); } // Init MediaSourceEngine. let mediaSourceEngine = this.playerInterface_.mediaSourceEngine; let forceTransmuxTS = this.config_.forceTransmuxTS; await mediaSourceEngine.init(streamsByType, forceTransmuxTS); if (this.destroyed_) { return; } this.setDuration_(); // Setup the initial set of Streams and then begin each update cycle. After // startup completes onUpdate_() will set up the remaining Periods. await this.setupStreams_(streams); if (this.destroyed_) { return; } streamsByType.forEach((stream, type) => { if (!this.mediaStates_.has(type)) { const state = this.createMediaState_(stream, needPeriodIndex, resumeAt); this.mediaStates_.set(type, state); this.scheduleUpdate_(state, 0); } }); }; /** * Creates a media state. * * @param {shaka.extern.Stream} stream * @param {number} needPeriodIndex * @param {number} resumeAt * @return {shaka.media.StreamingEngine.MediaState_} * @private */ shaka.media.StreamingEngine.prototype.createMediaState_ = function( stream, needPeriodIndex, resumeAt) { return /** @type {shaka.media.StreamingEngine.MediaState_} */ ({ stream: stream, type: stream.type, lastStream: null, lastSegmentReference: null, restoreStreamAfterTrickPlay: null, needInitSegment: true, needPeriodIndex: needPeriodIndex, endOfStream: false, performingUpdate: false, updateTimer: null, waitingToClearBuffer: false, clearBufferSafeMargin: 0, waitingToFlushBuffer: false, clearingBuffer: false, recovering: false, hasError: false, resumeAt: resumeAt || 0, operation: null, }); }; /** * Sets up the given Period if necessary. Calls onError() if an error occurs. * * @param {number} periodIndex The Period's index. * @return {!Promise} A Promise which resolves when the given Period is set up. * @private */ shaka.media.StreamingEngine.prototype.setupPeriod_ = function(periodIndex) { let canSwitchRecord = this.canSwitchPeriod_[periodIndex]; if (canSwitchRecord) { shaka.log.debug( '(all) Period ' + periodIndex + ' is being or has been set up'); goog.asserts.assert(canSwitchRecord.promise, 'promise must not be null'); return canSwitchRecord.promise; } shaka.log.debug('(all) setting up Period ' + periodIndex); canSwitchRecord = { promise: new shaka.util.PublicPromise(), resolved: false, }; this.canSwitchPeriod_[periodIndex] = canSwitchRecord; const streams = new Set(); // Add all video, trick video, and audio streams. for (const variant of this.manifest_.periods[periodIndex].variants) { if (variant.video) { streams.add(variant.video); } if (variant.video && variant.video.trickModeVideo) { streams.add(variant.video.trickModeVideo); } if (variant.audio) { streams.add(variant.audio); } } // Add text streams for (const stream of this.manifest_.periods[periodIndex].textStreams) { streams.add(stream); } // Serialize Period set up. this.setupPeriodPromise_ = this.setupPeriodPromise_.then(function() { if (this.destroyed_) return; return this.setupStreams_(streams); }.bind(this)).then(function() { if (this.destroyed_) return; this.canSwitchPeriod_[periodIndex].promise.resolve(); this.canSwitchPeriod_[periodIndex].resolved = true; shaka.log.v1('(all) setup Period ' + periodIndex); }.bind(this)).catch(function(error) { if (this.destroyed_) return; this.canSwitchPeriod_[periodIndex].promise.catch(() => {}); this.canSwitchPeriod_[periodIndex].promise.reject(); delete this.canSwitchPeriod_[periodIndex]; shaka.log.warning('(all) failed to setup Period ' + periodIndex); this.playerInterface_.onError(error); // Don't stop other Periods from being set up. }.bind(this)); return canSwitchRecord.promise; }; /** * Sets up the given Streams if necessary. Does NOT call onError() if an * error occurs. * * @param {!Set.<!shaka.extern.Stream>} streams * Use a set instead of list because duplicate ids will cause the player to * hang. * @return {!Promise} * @private */ shaka.media.StreamingEngine.prototype.setupStreams_ = async function(streams) { // Parallelize Stream set up. const parallelWork = []; for (const stream of streams) { const canSwitchRecord = this.canSwitchStream_.get(stream.id); if (canSwitchRecord) { shaka.log.debug( '(all) Stream ' + stream.id + ' is being or has been set up'); parallelWork.push(canSwitchRecord.promise); } else { shaka.log.v1('(all) setting up Stream ' + stream.id); this.canSwitchStream_.set(stream.id, { promise: new shaka.util.PublicPromise(), resolved: false, }); parallelWork.push(stream.createSegmentIndex()); } } try { await Promise.all(parallelWork); if (this.destroyed_) return; } catch (error) { if (this.destroyed_) return; for (const stream of streams) { this.canSwitchStream_.get(stream.id).promise.catch(() => {}); this.canSwitchStream_.get(stream.id).promise.reject(); this.canSwitchStream_.delete(stream.id); } throw error; } for (const stream of streams) { const canSwitchRecord = this.canSwitchStream_.get(stream.id); if (!canSwitchRecord.resolved) { canSwitchRecord.promise.resolve(); canSwitchRecord.resolved = true; shaka.log.v1('(all) setup Stream ' + stream.id); } } }; /** * Sets the MediaSource's duration. * @private */ shaka.media.StreamingEngine.prototype.setDuration_ = function() { let duration = this.manifest_.presentationTimeline.getDuration(); if (duration < Infinity) { this.playerInterface_.mediaSourceEngine.setDuration(duration); } else { // Not all platforms support infinite durations, so set a finite duration // so we can append segments and so the user agent can seek. this.playerInterface_.mediaSourceEngine.setDuration(Math.pow(2, 32)); } }; /** * Called when |mediaState|'s update timer has expired. * * @param {!shaka.media.StreamingEngine.MediaState_} mediaState * @private */ shaka.media.StreamingEngine.prototype.onUpdate_ = function(mediaState) { if (this.destroyed_) return; let logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState); // Sanity check. goog.asserts.assert( !mediaState.performingUpdate && (mediaState.updateTimer != null), logPrefix + ' unexpected call to onUpdate_()'); if (mediaState.performingUpdate || (mediaState.updateTimer == null)) return; goog.asserts.assert( !mediaState.clearingBuffer, logPrefix + ' onUpdate_() should not be called when clearing the buffer'); if (mediaState.clearingBuffer) return; mediaState.updateTimer = null; // Handle pending buffer clears. if (mediaState.waitingToClearBuffer) { // Note: clearBuffer_() will schedule the next update. shaka.log.debug(logPrefix, 'skipping update and clearing the buffer'); this.clearBuffer_( mediaState, mediaState.waitingToFlushBuffer, mediaState.clearBufferSafeMargin); return; } // Update the MediaState. try { let delay = this.update_(mediaState); if (delay != null) { this.scheduleUpdate_(mediaState, delay); mediaState.hasError = false; } } catch (error) { this.handleStreamingError_(error); return; } const mediaStates = Array.from(this.mediaStates_.values()); // Check if we've buffered to the end of the Period. this.handlePeriodTransition_(mediaState); // Check if we've buffered to the end of the presentation. We delay adding // the audio and video media states, so it is possible for the text stream // to be the only state and buffer to the end. So we need to wait until we // have completed startup to determine if we have reached the end. if (this.startupComplete_ && mediaStates.every(function(ms) { return ms.endOfStream; })) { shaka.log.v1(logPrefix, 'calling endOfStream()...'); this.playerInterface_.mediaSourceEngine.endOfStream().then(function() { if (this.destroyed_) { return; } // If the media segments don't reach the end, then we need to update the // timeline duration to match the final media duration to avoid buffering // forever at the end. We should only do this if the duration needs to // shrink. Growing it by less than 1ms can actually cause buffering on // replay, as in https://github.com/google/shaka-player/issues/979 // On some platforms, this can spuriously be 0, so ignore this case. // https://github.com/google/shaka-player/issues/1967, const duration = this.playerInterface_.mediaSourceEngine.getDuration(); if (duration != 0 && duration < this.manifest_.presentationTimeline.getDuration()) { this.manifest_.presentationTimeline.setDuration(duration); } }.bind(this)); } }; /** * Updates the given MediaState. * * @param {shaka.media.StreamingEngine.MediaState_} mediaState * @return {?number} The number of seconds to wait until updating again or * null if another update does not need to be scheduled. * @throws {!shaka.util.Error} if an error occurs. * @private */ shaka.media.StreamingEngine.prototype.update_ = function(mediaState) { goog.asserts.assert(this.manifest_, 'manifest_ should not be null'); goog.asserts.assert(this.config_, 'config_ should not be null'); const ContentType = shaka.util.ManifestParserUtils.ContentType; // If it's a text stream and the original id starts with 'CC', it's CEA // closed captions. Do not schedule update for closed captions text // mediastate, since closed captions are embedded in video streams. if (shaka.media.StreamingEngine.isEmbeddedText_(mediaState)) { this.playerInterface_.mediaSourceEngine.setSelectedClosedCaptionId( mediaState.stream.originalId || ''); return null; } let logPrefix = shaka.media.StreamingEngine.logPrefix_(mediaState); // Compute how far we've buffered ahead of the playhead. const presentationTime = this.playerInterface_.getPresentationTime(); // Get the next timestamp we need. let timeNeeded = this.getTimeNeeded_(mediaState, presentationTime); shaka.log.v2(logPrefix, 'timeNeeded=' + timeNeeded); let currentPeriodIndex = this.findPeriodContainingStream_(mediaState.stream); const needPeriodIndex = this.findPeriodForTime_(timeNeeded); // Get the amount of content we have buffered, accounting for drift. This // is only used to determine if we have meet the buffering goal. This should // be the same method that PlayheadObserver uses. let bufferedAhead = this.playerInterface_.mediaSourceEngine.bufferedAheadOf( mediaState.type, presentationTime); shaka.log.v2(logPrefix, 'update_:', 'presentationTime=' + presentationTime, 'bufferedAhead=' + bufferedAhead); let unscaledBufferingGoal = Math.max( this.manifest_.minBufferTime || 0, this.config_.rebufferingGoal, this.con