summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaDecoderStateMachine.cpp
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /dom/media/MediaDecoderStateMachine.cpp
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'dom/media/MediaDecoderStateMachine.cpp')
-rw-r--r--dom/media/MediaDecoderStateMachine.cpp3344
1 files changed, 3344 insertions, 0 deletions
diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp
new file mode 100644
index 000000000..f13e59b6c
--- /dev/null
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -0,0 +1,3344 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef XP_WIN
+// Include Windows headers required for enabling high precision timers.
+#include "windows.h"
+#include "mmsystem.h"
+#endif
+
+#include <algorithm>
+#include <stdint.h>
+
+#include "gfx2DGlue.h"
+
+#include "mediasink/AudioSinkWrapper.h"
+#include "mediasink/DecodedAudioDataSink.h"
+#include "mediasink/DecodedStream.h"
+#include "mediasink/OutputStreamManager.h"
+#include "mediasink/VideoSink.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Logging.h"
+#include "mozilla/mozalloc.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/SharedThreadPool.h"
+#include "mozilla/TaskQueue.h"
+
+#include "nsComponentManagerUtils.h"
+#include "nsContentUtils.h"
+#include "nsIEventTarget.h"
+#include "nsITimer.h"
+#include "nsPrintfCString.h"
+#include "nsTArray.h"
+#include "nsDeque.h"
+#include "prenv.h"
+
+#include "AccurateSeekTask.h"
+#include "AudioSegment.h"
+#include "DOMMediaStream.h"
+#include "ImageContainer.h"
+#include "MediaDecoder.h"
+#include "MediaDecoderReader.h"
+#include "MediaDecoderReaderWrapper.h"
+#include "MediaDecoderStateMachine.h"
+#include "MediaShutdownManager.h"
+#include "MediaPrefs.h"
+#include "MediaTimer.h"
+#include "NextFrameSeekTask.h"
+#include "TimeUnits.h"
+#include "VideoSegment.h"
+#include "VideoUtils.h"
+#include "gfxPrefs.h"
+
+namespace mozilla {
+
+using namespace mozilla::dom;
+using namespace mozilla::layers;
+using namespace mozilla::media;
+
+#define NS_DispatchToMainThread(...) CompileError_UseAbstractThreadDispatchInstead
+
+// avoid redefined macro in unified build
+#undef FMT
+#undef DECODER_LOG
+#undef VERBOSE_LOG
+#undef SAMPLE_LOG
+#undef DECODER_WARN
+#undef DUMP_LOG
+#undef SFMT
+#undef SLOG
+#undef SWARN
+#undef SDUMP
+
+#define FMT(x, ...) "Decoder=%p " x, mDecoderID, ##__VA_ARGS__
+#define DECODER_LOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (FMT(x, ##__VA_ARGS__)))
+#define VERBOSE_LOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, (FMT(x, ##__VA_ARGS__)))
+#define SAMPLE_LOG(x, ...) MOZ_LOG(gMediaSampleLog, LogLevel::Debug, (FMT(x, ##__VA_ARGS__)))
+#define DECODER_WARN(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get())
+#define DUMP_LOG(x, ...) NS_DebugBreak(NS_DEBUG_WARNING, nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, nullptr, -1)
+
+// Used by StateObject and its sub-classes
+#define SFMT(x, ...) "Decoder=%p state=%s " x, mMaster->mDecoderID, ToStateStr(GetState()), ##__VA_ARGS__
+#define SLOG(x, ...) MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (SFMT(x, ##__VA_ARGS__)))
+#define SWARN(x, ...) NS_WARNING(nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get())
+#define SDUMP(x, ...) NS_DebugBreak(NS_DEBUG_WARNING, nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get(), nullptr, nullptr, -1)
+
+// Certain constants get stored as member variables and then adjusted by various
+// scale factors on a per-decoder basis. We want to make sure to avoid using these
+// constants directly, so we put them in a namespace.
+namespace detail {
+
+// If audio queue has less than this many usecs of decoded audio, we won't risk
+// trying to decode the video, we'll skip decoding video up to the next
+// keyframe. We may increase this value for an individual decoder if we
+// encounter video frames which take a long time to decode.
+static const uint32_t LOW_AUDIO_USECS = 300000;
+
+// If more than this many usecs of decoded audio is queued, we'll hold off
+// decoding more audio. If we increase the low audio threshold (see
+// LOW_AUDIO_USECS above) we'll also increase this value to ensure it's not
+// less than the low audio threshold.
+static const int64_t AMPLE_AUDIO_USECS = 2000000;
+
+} // namespace detail
+
+// If we have fewer than LOW_VIDEO_FRAMES decoded frames, and
+// we're not "prerolling video", we'll skip the video up to the next keyframe
+// which is at or after the current playback position.
+static const uint32_t LOW_VIDEO_FRAMES = 2;
+
+// Threshold in usecs that used to check if we are low on decoded video.
+// If the last video frame's end time |mDecodedVideoEndTime| is more than
+// |LOW_VIDEO_THRESHOLD_USECS*mPlaybackRate| after the current clock in
+// Advanceframe(), the video decode is lagging, and we skip to next keyframe.
+static const int32_t LOW_VIDEO_THRESHOLD_USECS = 60000;
+
+// Arbitrary "frame duration" when playing only audio.
+static const int AUDIO_DURATION_USECS = 40000;
+
+// If we increase our "low audio threshold" (see LOW_AUDIO_USECS above), we
+// use this as a factor in all our calculations. Increasing this will cause
+// us to be more likely to increase our low audio threshold, and to
+// increase it by more.
+static const int THRESHOLD_FACTOR = 2;
+
+namespace detail {
+
+// If we have less than this much undecoded data available, we'll consider
+// ourselves to be running low on undecoded data. We determine how much
+// undecoded data we have remaining using the reader's GetBuffered()
+// implementation.
+static const int64_t LOW_DATA_THRESHOLD_USECS = 5000000;
+
+// LOW_DATA_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS, otherwise
+// the skip-to-keyframe logic can activate when we're running low on data.
+static_assert(LOW_DATA_THRESHOLD_USECS > AMPLE_AUDIO_USECS,
+ "LOW_DATA_THRESHOLD_USECS is too small");
+
+} // namespace detail
+
+// Amount of excess usecs of data to add in to the "should we buffer" calculation.
+static const uint32_t EXHAUSTED_DATA_MARGIN_USECS = 100000;
+
+static int64_t DurationToUsecs(TimeDuration aDuration) {
+ return static_cast<int64_t>(aDuration.ToSeconds() * USECS_PER_S);
+}
+
+static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3;
+static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10;
+#ifdef MOZ_APPLEMEDIA
+static const uint32_t HW_VIDEO_QUEUE_SIZE = 10;
+#else
+static const uint32_t HW_VIDEO_QUEUE_SIZE = 3;
+#endif
+static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999;
+
+static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE;
+static uint32_t sVideoQueueHWAccelSize = HW_VIDEO_QUEUE_SIZE;
+static uint32_t sVideoQueueSendToCompositorSize = VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE;
+
+static void InitVideoQueuePrefs() {
+ MOZ_ASSERT(NS_IsMainThread());
+ static bool sPrefInit = false;
+ if (!sPrefInit) {
+ sPrefInit = true;
+ sVideoQueueDefaultSize = Preferences::GetUint(
+ "media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE);
+ sVideoQueueHWAccelSize = Preferences::GetUint(
+ "media.video-queue.hw-accel-size", HW_VIDEO_QUEUE_SIZE);
+ sVideoQueueSendToCompositorSize = Preferences::GetUint(
+ "media.video-queue.send-to-compositor-size", VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE);
+ }
+}
+
+// Delay, in milliseconds, that tabs needs to be in background before video
+// decoding is suspended.
+static TimeDuration
+SuspendBackgroundVideoDelay()
+{
+ return TimeDuration::FromMilliseconds(
+ MediaPrefs::MDSMSuspendBackgroundVideoDelay());
+}
+
+class MediaDecoderStateMachine::StateObject
+{
+public:
+ virtual ~StateObject() {}
+ virtual void Exit() {}; // Exit action.
+ virtual void Step() {} // Perform a 'cycle' of this state object.
+ virtual State GetState() const = 0;
+
+ // Event handlers for various events.
+ virtual void HandleCDMProxyReady() {}
+ virtual void HandleAudioDecoded(MediaData* aAudio) {}
+ virtual void HandleVideoDecoded(MediaData* aVideo, TimeStamp aDecodeStart) {}
+ virtual void HandleEndOfStream() {}
+ virtual void HandleWaitingForData() {}
+ virtual void HandleAudioCaptured() {}
+
+ virtual RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget);
+
+ virtual RefPtr<ShutdownPromise> HandleShutdown();
+
+ virtual void HandleVideoSuspendTimeout() = 0;
+
+ virtual void HandleResumeVideoDecoding();
+
+ virtual void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) {}
+
+ virtual void DumpDebugInfo() {}
+
+private:
+ template <class S, typename R, typename... As>
+ auto ReturnTypeHelper(R(S::*)(As...)) -> R;
+
+protected:
+ enum class EventVisibility : int8_t
+ {
+ Observable,
+ Suppressed
+ };
+
+ using Master = MediaDecoderStateMachine;
+ explicit StateObject(Master* aPtr) : mMaster(aPtr) {}
+ TaskQueue* OwnerThread() const { return mMaster->mTaskQueue; }
+ MediaResource* Resource() const { return mMaster->mResource; }
+ MediaDecoderReaderWrapper* Reader() const { return mMaster->mReader; }
+ const MediaInfo& Info() const { return mMaster->Info(); }
+ bool IsExpectingMoreData() const
+ {
+ // We are expecting more data if either the resource states so, or if we
+ // have a waiting promise pending (such as with non-MSE EME).
+ return Resource()->IsExpectingMoreData() ||
+ (Reader()->IsWaitForDataSupported() &&
+ (Reader()->IsWaitingAudioData() || Reader()->IsWaitingVideoData()));
+ }
+
+ // Note this function will delete the current state object.
+ // Don't access members to avoid UAF after this call.
+ template <class S, typename... Ts>
+ auto SetState(Ts... aArgs)
+ -> decltype(ReturnTypeHelper(&S::Enter))
+ {
+ // keep mMaster in a local object because mMaster will become invalid after
+ // the current state object is deleted.
+ auto master = mMaster;
+
+ auto s = new S(master);
+
+ MOZ_ASSERT(master->mState != s->GetState() ||
+ master->mState == DECODER_STATE_SEEKING);
+
+ SLOG("change state to: %s", ToStateStr(s->GetState()));
+
+ Exit();
+
+ master->mState = s->GetState();
+ master->mStateObj.reset(s);
+ return s->Enter(Move(aArgs)...);
+ }
+
+ // Take a raw pointer in order not to change the life cycle of MDSM.
+ // It is guaranteed to be valid by MDSM.
+ Master* mMaster;
+};
+
+/**
+ * Purpose: decode metadata like duration and dimensions of the media resource.
+ *
+ * Transition to other states when decoding metadata is done:
+ * SHUTDOWN if failing to decode metadata.
+ * WAIT_FOR_CDM if the media is encrypted and CDM is not available.
+ * DECODING_FIRSTFRAME otherwise.
+ */
+class MediaDecoderStateMachine::DecodeMetadataState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit DecodeMetadataState(Master* aPtr) : StateObject(aPtr) {}
+
+ void Enter()
+ {
+ MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
+ MOZ_ASSERT(!mMetadataRequest.Exists());
+ SLOG("Dispatching AsyncReadMetadata");
+
+ // Set mode to METADATA since we are about to read metadata.
+ Resource()->SetReadMode(MediaCacheStream::MODE_METADATA);
+
+ // We disconnect mMetadataRequest in Exit() so it is fine to capture
+ // a raw pointer here.
+ mMetadataRequest.Begin(Reader()->ReadMetadata()
+ ->Then(OwnerThread(), __func__,
+ [this] (MetadataHolder* aMetadata) {
+ OnMetadataRead(aMetadata);
+ },
+ [this] (const MediaResult& aError) {
+ OnMetadataNotRead(aError);
+ }));
+ }
+
+ void Exit() override
+ {
+ mMetadataRequest.DisconnectIfExists();
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_DECODING_METADATA;
+ }
+
+ RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
+ {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek while decoding metadata.");
+ return MediaDecoder::SeekPromise::CreateAndReject(true, __func__);
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ // Do nothing since no decoders are created yet.
+ }
+
+ void HandleResumeVideoDecoding() override
+ {
+ // We never suspend video decoding in this state.
+ MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
+ }
+
+private:
+ void OnMetadataRead(MetadataHolder* aMetadata);
+
+ void OnMetadataNotRead(const MediaResult& aError)
+ {
+ mMetadataRequest.Complete();
+ SWARN("Decode metadata failed, shutting down decoder");
+ mMaster->DecodeError(aError);
+ }
+
+ MozPromiseRequestHolder<MediaDecoderReader::MetadataPromise> mMetadataRequest;
+};
+
+/**
+ * Purpose: wait for the CDM to start decoding.
+ *
+ * Transition to other states when CDM is ready:
+ * DECODING_FIRSTFRAME otherwise.
+ */
+class MediaDecoderStateMachine::WaitForCDMState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit WaitForCDMState(Master* aPtr) : StateObject(aPtr) {}
+
+ void Enter()
+ {
+ MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
+ }
+
+ void Exit() override
+ {
+ // mPendingSeek is either moved in HandleCDMProxyReady() or should be
+ // rejected here before transition to SHUTDOWN.
+ mPendingSeek.RejectIfExists(__func__);
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_WAIT_FOR_CDM;
+ }
+
+ void HandleCDMProxyReady() override;
+
+ RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
+ {
+ SLOG("Not Enough Data to seek at this stage, queuing seek");
+ mPendingSeek.RejectIfExists(__func__);
+ mPendingSeek.mTarget = aTarget;
+ return mPendingSeek.mPromise.Ensure(__func__);
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ // Do nothing since no decoders are created yet.
+ }
+
+ void HandleResumeVideoDecoding() override
+ {
+ // We never suspend video decoding in this state.
+ MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
+ }
+
+private:
+ SeekJob mPendingSeek;
+};
+
+/**
+ * Purpose: release decoder resources to save memory and hardware resources.
+ *
+ * Transition to:
+ * SEEKING if any seek request or play state changes to PLAYING.
+ */
+class MediaDecoderStateMachine::DormantState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit DormantState(Master* aPtr) : StateObject(aPtr) {}
+
+ void Enter()
+ {
+ if (mMaster->IsPlaying()) {
+ mMaster->StopPlayback();
+ }
+
+ // Calculate the position to seek to when exiting dormant.
+ auto t = mMaster->mMediaSink->IsStarted()
+ ? mMaster->GetClock()
+ : mMaster->GetMediaTime();
+ mPendingSeek.mTarget = SeekTarget(t, SeekTarget::Accurate);
+ // SeekJob asserts |mTarget.IsValid() == !mPromise.IsEmpty()| so we
+ // need to create the promise even it is not used at all.
+ RefPtr<MediaDecoder::SeekPromise> x = mPendingSeek.mPromise.Ensure(__func__);
+
+ mMaster->Reset();
+ mMaster->mReader->ReleaseResources();
+ }
+
+ void Exit() override
+ {
+ // mPendingSeek is either moved when exiting dormant or
+ // should be rejected here before transition to SHUTDOWN.
+ mPendingSeek.RejectIfExists(__func__);
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_DORMANT;
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ // Do nothing since we've released decoders in Enter().
+ }
+
+ void HandleResumeVideoDecoding() override
+ {
+ // Do nothing since we won't resume decoding until exiting dormant.
+ }
+
+ void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override;
+
+private:
+ SeekJob mPendingSeek;
+};
+
+/**
+ * Purpose: decode the 1st audio and video frames to fire the 'loadeddata' event.
+ *
+ * Transition to:
+ * SHUTDOWN if any decode error.
+ * SEEKING if any pending seek and seek is possible.
+ * DECODING when the 'loadeddata' event is fired.
+ */
+class MediaDecoderStateMachine::DecodingFirstFrameState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit DecodingFirstFrameState(Master* aPtr) : StateObject(aPtr) {}
+
+ void Enter(SeekJob aPendingSeek);
+
+ void Exit() override
+ {
+ // mPendingSeek is either moved before transition to SEEKING,
+ // or should be rejected here before transition to SHUTDOWN.
+ mPendingSeek.RejectIfExists(__func__);
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_DECODING_FIRSTFRAME;
+ }
+
+ void HandleAudioDecoded(MediaData* aAudio) override
+ {
+ mMaster->Push(aAudio, MediaData::AUDIO_DATA);
+ MaybeFinishDecodeFirstFrame();
+ }
+
+ void HandleVideoDecoded(MediaData* aVideo, TimeStamp aDecodeStart) override
+ {
+ mMaster->Push(aVideo, MediaData::VIDEO_DATA);
+ MaybeFinishDecodeFirstFrame();
+ }
+
+ void HandleEndOfStream() override
+ {
+ MaybeFinishDecodeFirstFrame();
+ }
+
+ RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override;
+
+ void HandleVideoSuspendTimeout() override
+ {
+ // Do nothing for we need to decode the 1st video frame to get the dimensions.
+ }
+
+ void HandleResumeVideoDecoding() override
+ {
+ // We never suspend video decoding in this state.
+ MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
+ }
+
+private:
+ // Notify FirstFrameLoaded if having decoded first frames and
+ // transition to SEEKING if there is any pending seek, or DECODING otherwise.
+ void MaybeFinishDecodeFirstFrame();
+
+ SeekJob mPendingSeek;
+};
+
+/**
+ * Purpose: decode audio/video data for playback.
+ *
+ * Transition to:
+ * DORMANT if playback is paused for a while.
+ * SEEKING if any seek request.
+ * SHUTDOWN if any decode error.
+ * BUFFERING if playback can't continue due to lack of decoded data.
+ * COMPLETED when having decoded all audio/video data.
+ */
+class MediaDecoderStateMachine::DecodingState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit DecodingState(Master* aPtr)
+ : StateObject(aPtr)
+ , mDormantTimer(OwnerThread())
+ {
+ }
+
+ void Enter();
+
+ void Exit() override
+ {
+ if (!mDecodeStartTime.IsNull()) {
+ TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime;
+ SLOG("Exiting DECODING, decoded for %.3lfs", decodeDuration.ToSeconds());
+ }
+ mDormantTimer.Reset();
+ }
+
+ void Step() override
+ {
+ if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING &&
+ mMaster->IsPlaying()) {
+ // We're playing, but the element/decoder is in paused state. Stop
+ // playing!
+ mMaster->StopPlayback();
+ }
+
+ // Start playback if necessary so that the clock can be properly queried.
+ if (!mIsPrerolling) {
+ mMaster->MaybeStartPlayback();
+ }
+
+ mMaster->UpdatePlaybackPositionPeriodically();
+
+ MOZ_ASSERT(!mMaster->IsPlaying() ||
+ mMaster->IsStateMachineScheduled(),
+ "Must have timer scheduled");
+
+ MaybeStartBuffering();
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_DECODING;
+ }
+
+ void HandleAudioDecoded(MediaData* aAudio) override
+ {
+ mMaster->Push(aAudio, MediaData::AUDIO_DATA);
+ MaybeStopPrerolling();
+ }
+
+ void HandleVideoDecoded(MediaData* aVideo, TimeStamp aDecodeStart) override
+ {
+ mMaster->Push(aVideo, MediaData::VIDEO_DATA);
+ MaybeStopPrerolling();
+ CheckSlowDecoding(aDecodeStart);
+ }
+
+ void HandleEndOfStream() override;
+
+ void HandleWaitingForData() override
+ {
+ MaybeStopPrerolling();
+ }
+
+ void HandleAudioCaptured() override
+ {
+ MaybeStopPrerolling();
+ // MediaSink is changed. Schedule Step() to check if we can start playback.
+ mMaster->ScheduleStateMachine();
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ if (mMaster->HasVideo()) {
+ mMaster->mVideoDecodeSuspended = true;
+ mMaster->mOnPlaybackEvent.Notify(MediaEventType::EnterVideoSuspend);
+ Reader()->SetVideoBlankDecode(true);
+ }
+ }
+
+ void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override
+ {
+ if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
+ // Schedule Step() to check if we can start playback.
+ mMaster->ScheduleStateMachine();
+ }
+
+ if (aPlayState == MediaDecoder::PLAY_STATE_PAUSED) {
+ StartDormantTimer();
+ } else {
+ mDormantTimer.Reset();
+ }
+ }
+
+ void DumpDebugInfo() override
+ {
+ SDUMP("mIsPrerolling=%d", mIsPrerolling);
+ }
+
+private:
+ void MaybeStartBuffering();
+
+ void CheckSlowDecoding(TimeStamp aDecodeStart)
+ {
+ // For non async readers, if the requested video sample was slow to
+ // arrive, increase the amount of audio we buffer to ensure that we
+ // don't run out of audio. This is unnecessary for async readers,
+ // since they decode audio and video on different threads so they
+ // are unlikely to run out of decoded audio.
+ if (Reader()->IsAsync()) {
+ return;
+ }
+
+ TimeDuration decodeTime = TimeStamp::Now() - aDecodeStart;
+ int64_t adjustedTime = THRESHOLD_FACTOR * DurationToUsecs(decodeTime);
+ if (adjustedTime > mMaster->mLowAudioThresholdUsecs &&
+ !mMaster->HasLowBufferedData())
+ {
+ mMaster->mLowAudioThresholdUsecs =
+ std::min(adjustedTime, mMaster->mAmpleAudioThresholdUsecs);
+
+ mMaster->mAmpleAudioThresholdUsecs =
+ std::max(THRESHOLD_FACTOR * mMaster->mLowAudioThresholdUsecs,
+ mMaster->mAmpleAudioThresholdUsecs);
+
+ SLOG("Slow video decode, set "
+ "mLowAudioThresholdUsecs=%lld "
+ "mAmpleAudioThresholdUsecs=%lld",
+ mMaster->mLowAudioThresholdUsecs,
+ mMaster->mAmpleAudioThresholdUsecs);
+ }
+ }
+
+ bool DonePrerollingAudio()
+ {
+ return !mMaster->IsAudioDecoding() ||
+ mMaster->GetDecodedAudioDuration() >=
+ mMaster->AudioPrerollUsecs() * mMaster->mPlaybackRate;
+ }
+
+ bool DonePrerollingVideo()
+ {
+ return !mMaster->IsVideoDecoding() ||
+ static_cast<uint32_t>(mMaster->VideoQueue().GetSize()) >=
+ mMaster->VideoPrerollFrames() * mMaster->mPlaybackRate + 1;
+ }
+
+ void MaybeStopPrerolling()
+ {
+ if (mIsPrerolling &&
+ (DonePrerollingAudio() || Reader()->IsWaitingAudioData()) &&
+ (DonePrerollingVideo() || Reader()->IsWaitingVideoData())) {
+ mIsPrerolling = false;
+ // Check if we can start playback.
+ mMaster->ScheduleStateMachine();
+ }
+ }
+
+ void EnterDormant()
+ {
+ SetState<DormantState>();
+ }
+
+ void StartDormantTimer()
+ {
+ if (!mMaster->mMediaSeekable) {
+ // Don't enter dormant if the media is not seekable because we need to
+ // seek when exiting dormant.
+ return;
+ }
+
+ auto timeout = MediaPrefs::DormantOnPauseTimeout();
+ if (timeout < 0) {
+ // Disabled when timeout is negative.
+ return;
+ } else if (timeout == 0) {
+ // Enter dormant immediately without scheduling a timer.
+ EnterDormant();
+ return;
+ }
+
+ TimeStamp target = TimeStamp::Now() +
+ TimeDuration::FromMilliseconds(timeout);
+
+ mDormantTimer.Ensure(target,
+ [this] () {
+ mDormantTimer.CompleteRequest();
+ EnterDormant();
+ }, [this] () {
+ mDormantTimer.CompleteRequest();
+ });
+ }
+
+ // Time at which we started decoding.
+ TimeStamp mDecodeStartTime;
+
+ // When we start decoding (either for the first time, or after a pause)
+ // we may be low on decoded data. We don't want our "low data" logic to
+ // kick in and decide that we're low on decoded data because the download
+ // can't keep up with the decode, and cause us to pause playback. So we
+ // have a "preroll" stage, where we ignore the results of our "low data"
+ // logic during the first few frames of our decode. This occurs during
+ // playback.
+ bool mIsPrerolling = true;
+
+ // Fired when playback is paused for a while to enter dormant.
+ DelayedScheduler mDormantTimer;
+};
+
+/**
+ * Purpose: seek to a particular new playback position.
+ *
+ * Transition to:
+ * SEEKING if any new seek request.
+ * SHUTDOWN if seek failed.
+ * COMPLETED if the new playback position is the end of the media resource.
+ * DECODING otherwise.
+ */
+class MediaDecoderStateMachine::SeekingState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit SeekingState(Master* aPtr) : StateObject(aPtr) {}
+
+ RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob aSeekJob,
+ EventVisibility aVisibility)
+ {
+ mSeekJob = Move(aSeekJob);
+ mVisibility = aVisibility;
+
+ // Always switch off the blank decoder otherwise we might become visible
+ // in the middle of seeking and won't have a valid video frame to show
+ // when seek is done.
+ if (mMaster->mVideoDecodeSuspended) {
+ mMaster->mVideoDecodeSuspended = false;
+ mMaster->mOnPlaybackEvent.Notify(MediaEventType::ExitVideoSuspend);
+ Reader()->SetVideoBlankDecode(false);
+ }
+
+ // SeekTask will register its callbacks to MediaDecoderReaderWrapper.
+ mMaster->CancelMediaDecoderReaderWrapperCallback();
+
+ // Create a new SeekTask instance for the incoming seek task.
+ if (mSeekJob.mTarget.IsAccurate() ||
+ mSeekJob.mTarget.IsFast()) {
+ mSeekTask = new AccurateSeekTask(
+ mMaster->mDecoderID, OwnerThread(), Reader(), mSeekJob.mTarget,
+ Info(), mMaster->Duration(), mMaster->GetMediaTime());
+ } else if (mSeekJob.mTarget.IsNextFrame()) {
+ mSeekTask = new NextFrameSeekTask(
+ mMaster->mDecoderID, OwnerThread(), Reader(), mSeekJob.mTarget,
+ Info(), mMaster->Duration(),mMaster->GetMediaTime(),
+ mMaster->AudioQueue(), mMaster->VideoQueue());
+ } else {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Cannot handle this seek task.");
+ }
+
+ // Don't stop playback for a video-only seek since audio is playing.
+ if (!mSeekJob.mTarget.IsVideoOnly()) {
+ mMaster->StopPlayback();
+ }
+
+ // mSeekJob.mTarget.mTime might be different from
+ // mSeekTask->GetSeekTarget().mTime because the seek task might clamp the
+ // seek target to [0, duration]. We want to update the playback position to
+ // the clamped value.
+ mMaster->UpdatePlaybackPositionInternal(
+ mSeekTask->GetSeekTarget().GetTime().ToMicroseconds());
+
+ if (mVisibility == EventVisibility::Observable) {
+ mMaster->mOnPlaybackEvent.Notify(MediaEventType::SeekStarted);
+ // We want dormant actions to be transparent to the user.
+ // So we only notify the change when the seek request is from the user.
+ mMaster->UpdateNextFrameStatus(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING);
+ }
+
+ // Reset our state machine and decoding pipeline before seeking.
+ if (mSeekTask->NeedToResetMDSM()) {
+ if (mSeekJob.mTarget.IsVideoOnly()) {
+ mMaster->Reset(TrackInfo::kVideoTrack);
+ } else {
+ mMaster->Reset();
+ }
+ }
+
+ // Do the seek.
+ mSeekTaskRequest.Begin(mSeekTask->Seek(mMaster->Duration())
+ ->Then(OwnerThread(), __func__,
+ [this] (const SeekTaskResolveValue& aValue) {
+ OnSeekTaskResolved(aValue);
+ },
+ [this] (const SeekTaskRejectValue& aValue) {
+ OnSeekTaskRejected(aValue);
+ }));
+
+ return mSeekJob.mPromise.Ensure(__func__);
+ }
+
+ void Exit() override
+ {
+ mSeekTaskRequest.DisconnectIfExists();
+ mSeekJob.RejectIfExists(__func__);
+ mSeekTask->Discard();
+
+ // Reset the MediaDecoderReaderWrapper's callbask.
+ mMaster->SetMediaDecoderReaderWrapperCallback();
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_SEEKING;
+ }
+
+ void HandleAudioDecoded(MediaData* aAudio) override
+ {
+ MOZ_ASSERT(false);
+ }
+
+ void HandleVideoDecoded(MediaData* aVideo, TimeStamp aDecodeStart) override
+ {
+ MOZ_ASSERT(false);
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ // Do nothing since we want a valid video frame to show when seek is done.
+ }
+
+ void HandleResumeVideoDecoding() override
+ {
+ // We set mVideoDecodeSuspended to false in Enter().
+ MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
+ }
+
+private:
+ void OnSeekTaskResolved(const SeekTaskResolveValue& aValue)
+ {
+ mSeekTaskRequest.Complete();
+
+ if (aValue.mSeekedAudioData) {
+ mMaster->Push(aValue.mSeekedAudioData, MediaData::AUDIO_DATA);
+ mMaster->mDecodedAudioEndTime = std::max(
+ aValue.mSeekedAudioData->GetEndTime(), mMaster->mDecodedAudioEndTime);
+ }
+
+ if (aValue.mSeekedVideoData) {
+ mMaster->Push(aValue.mSeekedVideoData, MediaData::VIDEO_DATA);
+ mMaster->mDecodedVideoEndTime = std::max(
+ aValue.mSeekedVideoData->GetEndTime(), mMaster->mDecodedVideoEndTime);
+ }
+
+ if (aValue.mIsAudioQueueFinished) {
+ mMaster->AudioQueue().Finish();
+ }
+
+ if (aValue.mIsVideoQueueFinished) {
+ mMaster->VideoQueue().Finish();
+ }
+
+ SeekCompleted();
+ }
+
+ void OnSeekTaskRejected(const SeekTaskRejectValue& aValue)
+ {
+ mSeekTaskRequest.Complete();
+
+ if (aValue.mError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
+ mMaster->AudioQueue().Finish();
+ mMaster->VideoQueue().Finish();
+ SeekCompleted();
+ return;
+ }
+
+ if (aValue.mIsAudioQueueFinished) {
+ mMaster->AudioQueue().Finish();
+ }
+
+ if (aValue.mIsVideoQueueFinished) {
+ mMaster->VideoQueue().Finish();
+ }
+
+ mMaster->DecodeError(aValue.mError);
+ }
+
+ void SeekCompleted();
+
+ SeekJob mSeekJob;
+ EventVisibility mVisibility = EventVisibility::Observable;
+ MozPromiseRequestHolder<SeekTask::SeekTaskPromise> mSeekTaskRequest;
+ RefPtr<SeekTask> mSeekTask;
+};
+
+/**
+ * Purpose: stop playback until enough data is decoded to continue playback.
+ *
+ * Transition to:
+ * SEEKING if any seek request.
+ * SHUTDOWN if any decode error.
+ * COMPLETED when having decoded all audio/video data.
+ * DECODING when having decoded enough data to continue playback.
+ */
+class MediaDecoderStateMachine::BufferingState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit BufferingState(Master* aPtr) : StateObject(aPtr) {}
+
+ void Enter()
+ {
+ if (mMaster->IsPlaying()) {
+ mMaster->StopPlayback();
+ }
+
+ mBufferingStart = TimeStamp::Now();
+
+ MediaStatistics stats = mMaster->GetStatistics();
+ SLOG("Playback rate: %.1lfKB/s%s download rate: %.1lfKB/s%s",
+ stats.mPlaybackRate/1024, stats.mPlaybackRateReliable ? "" : " (unreliable)",
+ stats.mDownloadRate/1024, stats.mDownloadRateReliable ? "" : " (unreliable)");
+
+ mMaster->ScheduleStateMachineIn(USECS_PER_S);
+
+ mMaster->UpdateNextFrameStatus(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING);
+ }
+
+ void Step() override;
+
+ State GetState() const override
+ {
+ return DECODER_STATE_BUFFERING;
+ }
+
+ void HandleAudioDecoded(MediaData* aAudio) override
+ {
+ // This might be the sample we need to exit buffering.
+ // Schedule Step() to check it.
+ mMaster->Push(aAudio, MediaData::AUDIO_DATA);
+ mMaster->ScheduleStateMachine();
+ }
+
+ void HandleVideoDecoded(MediaData* aVideo, TimeStamp aDecodeStart) override
+ {
+ // This might be the sample we need to exit buffering.
+ // Schedule Step() to check it.
+ mMaster->Push(aVideo, MediaData::VIDEO_DATA);
+ mMaster->ScheduleStateMachine();
+ }
+
+ void HandleEndOfStream() override;
+
+ void HandleVideoSuspendTimeout() override
+ {
+ if (mMaster->HasVideo()) {
+ mMaster->mVideoDecodeSuspended = true;
+ mMaster->mOnPlaybackEvent.Notify(MediaEventType::EnterVideoSuspend);
+ Reader()->SetVideoBlankDecode(true);
+ }
+ }
+
+private:
+ TimeStamp mBufferingStart;
+
+ // The maximum number of second we spend buffering when we are short on
+ // unbuffered data.
+ const uint32_t mBufferingWait = 15;
+};
+
+/**
+ * Purpose: play all the decoded data and fire the 'ended' event.
+ *
+ * Transition to:
+ * SEEKING if any seek request.
+ */
+class MediaDecoderStateMachine::CompletedState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit CompletedState(Master* aPtr) : StateObject(aPtr) {}
+
+ void Enter()
+ {
+ // We've decoded all samples. We don't need decoders anymore.
+ Reader()->ReleaseResources();
+
+ bool hasNextFrame = (!mMaster->HasAudio() || !mMaster->mAudioCompleted)
+ && (!mMaster->HasVideo() || !mMaster->mVideoCompleted);
+
+ mMaster->UpdateNextFrameStatus(hasNextFrame
+ ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE
+ : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE);
+
+ mMaster->ScheduleStateMachine();
+ }
+
+ void Exit() override
+ {
+ mSentPlaybackEndedEvent = false;
+ }
+
+ void Step() override
+ {
+ if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING &&
+ mMaster->IsPlaying()) {
+ mMaster->StopPlayback();
+ }
+
+ // Play the remaining media. We want to run AdvanceFrame() at least
+ // once to ensure the current playback position is advanced to the
+ // end of the media, and so that we update the readyState.
+ if ((mMaster->HasVideo() && !mMaster->mVideoCompleted) ||
+ (mMaster->HasAudio() && !mMaster->mAudioCompleted)) {
+ // Start playback if necessary to play the remaining media.
+ mMaster->MaybeStartPlayback();
+ mMaster->UpdatePlaybackPositionPeriodically();
+ MOZ_ASSERT(!mMaster->IsPlaying() ||
+ mMaster->IsStateMachineScheduled(),
+ "Must have timer scheduled");
+ return;
+ }
+
+ // StopPlayback in order to reset the IsPlaying() state so audio
+ // is restarted correctly.
+ mMaster->StopPlayback();
+
+ if (mMaster->mPlayState == MediaDecoder::PLAY_STATE_PLAYING &&
+ !mSentPlaybackEndedEvent) {
+ int64_t clockTime = std::max(mMaster->AudioEndTime(), mMaster->VideoEndTime());
+ clockTime = std::max(int64_t(0), std::max(clockTime, mMaster->Duration().ToMicroseconds()));
+ mMaster->UpdatePlaybackPosition(clockTime);
+
+ // Ensure readyState is updated before firing the 'ended' event.
+ mMaster->UpdateNextFrameStatus(MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE);
+
+ mMaster->mOnPlaybackEvent.Notify(MediaEventType::PlaybackEnded);
+
+ mSentPlaybackEndedEvent = true;
+
+ // MediaSink::GetEndTime() must be called before stopping playback.
+ mMaster->StopMediaSink();
+ }
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_COMPLETED;
+ }
+
+ void HandleAudioCaptured() override
+ {
+ // MediaSink is changed. Schedule Step() to check if we can start playback.
+ mMaster->ScheduleStateMachine();
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ // Do nothing since no decoding is going on.
+ }
+
+ void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override
+ {
+ if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
+ // Schedule Step() to check if we can start playback.
+ mMaster->ScheduleStateMachine();
+ }
+ }
+
+private:
+ bool mSentPlaybackEndedEvent = false;
+};
+
+/**
+ * Purpose: release all resources allocated by MDSM.
+ *
+ * Transition to:
+ * None since this is the final state.
+ *
+ * Transition from:
+ * Any states other than SHUTDOWN.
+ */
+class MediaDecoderStateMachine::ShutdownState
+ : public MediaDecoderStateMachine::StateObject
+{
+public:
+ explicit ShutdownState(Master* aPtr) : StateObject(aPtr) {}
+
+ RefPtr<ShutdownPromise> Enter();
+
+ void Exit() override
+ {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Shouldn't escape the SHUTDOWN state.");
+ }
+
+ State GetState() const override
+ {
+ return DECODER_STATE_SHUTDOWN;
+ }
+
+ RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override
+ {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek in shutdown state.");
+ return MediaDecoder::SeekPromise::CreateAndReject(true, __func__);
+ }
+
+ RefPtr<ShutdownPromise> HandleShutdown() override
+ {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down.");
+ return nullptr;
+ }
+
+ void HandleVideoSuspendTimeout() override
+ {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down.");
+ }
+
+ void HandleResumeVideoDecoding() override
+ {
+ MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down.");
+ }
+};
+
+RefPtr<MediaDecoder::SeekPromise>
+MediaDecoderStateMachine::
+StateObject::HandleSeek(SeekTarget aTarget)
+{
+ SLOG("Changed state to SEEKING (to %lld)", aTarget.GetTime().ToMicroseconds());
+ SeekJob seekJob;
+ seekJob.mTarget = aTarget;
+ return SetState<SeekingState>(Move(seekJob), EventVisibility::Observable);
+}
+
+RefPtr<ShutdownPromise>
+MediaDecoderStateMachine::
+StateObject::HandleShutdown()
+{
+ return SetState<ShutdownState>();
+}
+
+static void
+ReportRecoveryTelemetry(const TimeStamp& aRecoveryStart,
+ const MediaInfo& aMediaInfo,
+ bool aIsHardwareAccelerated)
+{
+ MOZ_ASSERT(NS_IsMainThread());
+ if (!aMediaInfo.HasVideo()) {
+ return;
+ }
+
+ // Keyed by audio+video or video alone, hardware acceleration,
+ // and by a resolution range.
+ nsCString key(aMediaInfo.HasAudio() ? "AV" : "V");
+ key.AppendASCII(aIsHardwareAccelerated ? "(hw)," : ",");
+ static const struct { int32_t mH; const char* mRes; } sResolutions[] = {
+ { 240, "0-240" },
+ { 480, "241-480" },
+ { 720, "481-720" },
+ { 1080, "721-1080" },
+ { 2160, "1081-2160" }
+ };
+ const char* resolution = "2161+";
+ int32_t height = aMediaInfo.mVideo.mImage.height;
+ for (const auto& res : sResolutions) {
+ if (height <= res.mH) {
+ resolution = res.mRes;
+ break;
+ }
+ }
+ key.AppendASCII(resolution);
+
+ TimeDuration duration = TimeStamp::Now() - aRecoveryStart;
+ double duration_ms = duration.ToMilliseconds();
+ Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS,
+ key,
+ uint32_t(duration_ms + 0.5));
+ Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS,
+ NS_LITERAL_CSTRING("All"),
+ uint32_t(duration_ms + 0.5));
+}
+
+void
+MediaDecoderStateMachine::
+StateObject::HandleResumeVideoDecoding()
+{
+ MOZ_ASSERT(mMaster->mVideoDecodeSuspended);
+
+ // Start counting recovery time from right now.
+ TimeStamp start = TimeStamp::Now();
+
+ // Local reference to mInfo, so that it will be copied in the lambda below.
+ auto& info = Info();
+ bool hw = Reader()->VideoIsHardwareAccelerated();
+
+ // Start video-only seek to the current time.
+ SeekJob seekJob;
+
+ const SeekTarget::Type type = mMaster->HasAudio()
+ ? SeekTarget::Type::Accurate
+ : SeekTarget::Type::PrevSyncPoint;
+
+ seekJob.mTarget = SeekTarget(mMaster->GetMediaTime(),
+ type,
+ true /* aVideoOnly */);
+
+ SetState<SeekingState>(Move(seekJob), EventVisibility::Suppressed)->Then(
+ AbstractThread::MainThread(), __func__,
+ [start, info, hw](){ ReportRecoveryTelemetry(start, info, hw); },
+ [](){});
+}
+
+void
+MediaDecoderStateMachine::
+DecodeMetadataState::OnMetadataRead(MetadataHolder* aMetadata)
+{
+ mMetadataRequest.Complete();
+
+ // Set mode to PLAYBACK after reading metadata.
+ Resource()->SetReadMode(MediaCacheStream::MODE_PLAYBACK);
+
+ mMaster->mInfo = Some(aMetadata->mInfo);
+ mMaster->mMetadataTags = aMetadata->mTags.forget();
+ mMaster->mMediaSeekable = Info().mMediaSeekable;
+ mMaster->mMediaSeekableOnlyInBufferedRanges = Info().mMediaSeekableOnlyInBufferedRanges;
+
+ if (Info().mMetadataDuration.isSome()) {
+ mMaster->RecomputeDuration();
+ } else if (Info().mUnadjustedMetadataEndTime.isSome()) {
+ RefPtr<Master> master = mMaster;
+ Reader()->AwaitStartTime()->Then(OwnerThread(), __func__,
+ [master] () {
+ NS_ENSURE_TRUE_VOID(!master->IsShutdown());
+ auto& info = master->mInfo.ref();
+ TimeUnit unadjusted = info.mUnadjustedMetadataEndTime.ref();
+ TimeUnit adjustment = master->mReader->StartTime();
+ info.mMetadataDuration.emplace(unadjusted - adjustment);
+ master->RecomputeDuration();
+ }, [master, this] () {
+ SWARN("Adjusting metadata end time failed");
+ }
+ );
+ }
+
+ if (mMaster->HasVideo()) {
+ SLOG("Video decode isAsync=%d HWAccel=%d videoQueueSize=%d",
+ Reader()->IsAsync(),
+ Reader()->VideoIsHardwareAccelerated(),
+ mMaster->GetAmpleVideoFrames());
+ }
+
+ // In general, we wait until we know the duration before notifying the decoder.
+ // However, we notify unconditionally in this case without waiting for the start
+ // time, since the caller might be waiting on metadataloaded to be fired before
+ // feeding in the CDM, which we need to decode the first frame (and
+ // thus get the metadata). We could fix this if we could compute the start
+ // time by demuxing without necessaring decoding.
+ bool waitingForCDM = Info().IsEncrypted() && !mMaster->mCDMProxy;
+
+ mMaster->mNotifyMetadataBeforeFirstFrame =
+ mMaster->mDuration.Ref().isSome() || waitingForCDM;
+
+ if (mMaster->mNotifyMetadataBeforeFirstFrame) {
+ mMaster->EnqueueLoadedMetadataEvent();
+ }
+
+ if (waitingForCDM) {
+ // Metadata parsing was successful but we're still waiting for CDM caps
+ // to become available so that we can build the correct decryptor/decoder.
+ SetState<WaitForCDMState>();
+ } else {
+ SetState<DecodingFirstFrameState>(SeekJob{});
+ }
+}
+
+void
+MediaDecoderStateMachine::
+DormantState::HandlePlayStateChanged(MediaDecoder::PlayState aPlayState)
+{
+ if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
+ // Exit dormant when the user wants to play.
+ MOZ_ASSERT(!Info().IsEncrypted() || mMaster->mCDMProxy);
+ MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent);
+ SetState<SeekingState>(Move(mPendingSeek), EventVisibility::Suppressed);
+ }
+}
+
+void
+MediaDecoderStateMachine::
+WaitForCDMState::HandleCDMProxyReady()
+{
+ SetState<DecodingFirstFrameState>(Move(mPendingSeek));
+}
+
+void
+MediaDecoderStateMachine::
+DecodingFirstFrameState::Enter(SeekJob aPendingSeek)
+{
+ // Handle pending seek.
+ if (aPendingSeek.Exists() &&
+ (mMaster->mSentFirstFrameLoadedEvent ||
+ Reader()->ForceZeroStartTime())) {
+ SetState<SeekingState>(Move(aPendingSeek), EventVisibility::Observable);
+ return;
+ }
+
+ // Transition to DECODING if we've decoded first frames.
+ if (mMaster->mSentFirstFrameLoadedEvent) {
+ SetState<DecodingState>();
+ return;
+ }
+
+ MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
+
+ mPendingSeek = Move(aPendingSeek);
+
+ // Dispatch tasks to decode first frames.
+ mMaster->DispatchDecodeTasksIfNeeded();
+}
+
+RefPtr<MediaDecoder::SeekPromise>
+MediaDecoderStateMachine::
+DecodingFirstFrameState::HandleSeek(SeekTarget aTarget)
+{
+ // Should've transitioned to DECODING in Enter()
+ // if mSentFirstFrameLoadedEvent is true.
+ MOZ_ASSERT(!mMaster->mSentFirstFrameLoadedEvent);
+
+ if (!Reader()->ForceZeroStartTime()) {
+ SLOG("Not Enough Data to seek at this stage, queuing seek");
+ mPendingSeek.RejectIfExists(__func__);
+ mPendingSeek.mTarget = aTarget;
+ return mPendingSeek.mPromise.Ensure(__func__);
+ }
+
+ // Since ForceZeroStartTime() is true, we should've transitioned to SEEKING
+ // in Enter() if there is any pending seek.
+ MOZ_ASSERT(!mPendingSeek.Exists());
+
+ return StateObject::HandleSeek(aTarget);
+}
+
+void
+MediaDecoderStateMachine::
+DecodingFirstFrameState::MaybeFinishDecodeFirstFrame()
+{
+ MOZ_ASSERT(!mMaster->mSentFirstFrameLoadedEvent);
+
+ if ((mMaster->IsAudioDecoding() && mMaster->AudioQueue().GetSize() == 0) ||
+ (mMaster->IsVideoDecoding() && mMaster->VideoQueue().GetSize() == 0)) {
+ return;
+ }
+
+ mMaster->FinishDecodeFirstFrame();
+
+ if (mPendingSeek.Exists()) {
+ SetState<SeekingState>(Move(mPendingSeek), EventVisibility::Observable);
+ } else {
+ SetState<DecodingState>();
+ }
+}
+
+void
+MediaDecoderStateMachine::
+DecodingState::Enter()
+{
+ MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent);
+
+ if (!mMaster->mIsVisible &&
+ !mMaster->mVideoDecodeSuspendTimer.IsScheduled() &&
+ !mMaster->mVideoDecodeSuspended) {
+ // If we are not visible and the timer is not schedule, it means the timer
+ // has timed out and we should suspend video decoding now if necessary.
+ HandleVideoSuspendTimeout();
+ }
+
+ if (mMaster->CheckIfDecodeComplete()) {
+ SetState<CompletedState>();
+ return;
+ }
+
+ mMaster->UpdateNextFrameStatus(MediaDecoderOwner::NEXT_FRAME_AVAILABLE);
+
+ mDecodeStartTime = TimeStamp::Now();
+
+ MaybeStopPrerolling();
+
+ // Ensure that we've got tasks enqueued to decode data if we need to.
+ mMaster->DispatchDecodeTasksIfNeeded();
+
+ mMaster->ScheduleStateMachine();
+
+ // Will enter dormant when playback is paused for a while.
+ if (mMaster->mPlayState == MediaDecoder::PLAY_STATE_PAUSED) {
+ StartDormantTimer();
+ }
+}
+
+void
+MediaDecoderStateMachine::
+DecodingState::HandleEndOfStream()
+{
+ if (mMaster->CheckIfDecodeComplete()) {
+ SetState<CompletedState>();
+ } else {
+ MaybeStopPrerolling();
+ }
+}
+
+void
+MediaDecoderStateMachine::
+DecodingState::MaybeStartBuffering()
+{
+ // Buffering makes senses only after decoding first frames.
+ MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent);
+
+ // Don't enter buffering when MediaDecoder is not playing.
+ if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING) {
+ return;
+ }
+
+ // Don't enter buffering while prerolling so that the decoder has a chance to
+ // enqueue some decoded data before we give up and start buffering.
+ if (!mMaster->IsPlaying()) {
+ return;
+ }
+
+ bool shouldBuffer;
+ if (Reader()->UseBufferingHeuristics()) {
+ shouldBuffer = IsExpectingMoreData() &&
+ mMaster->HasLowDecodedData() &&
+ mMaster->HasLowBufferedData();
+ } else {
+ MOZ_ASSERT(Reader()->IsWaitForDataSupported());
+ shouldBuffer =
+ (mMaster->OutOfDecodedAudio() && Reader()->IsWaitingAudioData()) ||
+ (mMaster->OutOfDecodedVideo() && Reader()->IsWaitingVideoData());
+ }
+ if (shouldBuffer) {
+ SetState<BufferingState>();
+ }
+}
+
+void
+MediaDecoderStateMachine::
+SeekingState::SeekCompleted()
+{
+ int64_t seekTime = mSeekTask->GetSeekTarget().GetTime().ToMicroseconds();
+ int64_t newCurrentTime = seekTime;
+
+ // Setup timestamp state.
+ RefPtr<MediaData> video = mMaster->VideoQueue().PeekFront();
+ if (seekTime == mMaster->Duration().ToMicroseconds()) {
+ newCurrentTime = seekTime;
+ } else if (mMaster->HasAudio()) {
+ RefPtr<MediaData> audio = mMaster->AudioQueue().PeekFront();
+ // Though we adjust the newCurrentTime in audio-based, and supplemented
+ // by video. For better UX, should NOT bind the slide position to
+ // the first audio data timestamp directly.
+ // While seeking to a position where there's only either audio or video, or
+ // seeking to a position lies before audio or video, we need to check if
+ // seekTime is bounded in suitable duration. See Bug 1112438.
+ int64_t audioStart = audio ? audio->mTime : seekTime;
+ // We only pin the seek time to the video start time if the video frame
+ // contains the seek time.
+ if (video && video->mTime <= seekTime && video->GetEndTime() > seekTime) {
+ newCurrentTime = std::min(audioStart, video->mTime);
+ } else {
+ newCurrentTime = audioStart;
+ }
+ } else {
+ newCurrentTime = video ? video->mTime : seekTime;
+ }
+
+ // Change state to DECODING or COMPLETED now.
+ bool isLiveStream = Resource()->IsLiveStream();
+ State nextState;
+ if (newCurrentTime == mMaster->Duration().ToMicroseconds() && !isLiveStream) {
+ // Seeked to end of media, move to COMPLETED state. Note we don't do
+ // this when playing a live stream, since the end of media will advance
+ // once we download more data!
+ // Explicitly set our state so we don't decode further, and so
+ // we report playback ended to the media element.
+ nextState = DECODER_STATE_COMPLETED;
+ } else {
+ nextState = DECODER_STATE_DECODING;
+ }
+
+ // We want to resolve the seek request prior finishing the first frame
+ // to ensure that the seeked event is fired prior loadeded.
+ mSeekJob.Resolve(nextState == DECODER_STATE_COMPLETED, __func__);
+
+ // Notify FirstFrameLoaded now if we haven't since we've decoded some data
+ // for readyState to transition to HAVE_CURRENT_DATA and fire 'loadeddata'.
+ if (!mMaster->mSentFirstFrameLoadedEvent) {
+ // Only MSE can start seeking before finishing decoding first frames.
+ MOZ_ASSERT(Reader()->ForceZeroStartTime());
+ mMaster->FinishDecodeFirstFrame();
+ }
+
+ // Ensure timestamps are up to date.
+ if (!mSeekJob.mTarget.IsVideoOnly()) {
+ // Don't update playback position for video-only seek.
+ // Otherwise we might have |newCurrentTime > mMediaSink->GetPosition()|
+ // and fail the assertion in GetClock() since we didn't stop MediaSink.
+ mMaster->UpdatePlaybackPositionInternal(newCurrentTime);
+ }
+
+ // Try to decode another frame to detect if we're at the end...
+ SLOG("Seek completed, mCurrentPosition=%lld", mMaster->mCurrentPosition.Ref());
+
+ if (video) {
+ mMaster->mMediaSink->Redraw(Info().mVideo);
+ mMaster->mOnPlaybackEvent.Notify(MediaEventType::Invalidate);
+ }
+
+ if (nextState == DECODER_STATE_COMPLETED) {
+ SetState<CompletedState>();
+ } else {
+ SetState<DecodingState>();
+ }
+}
+
+void
+MediaDecoderStateMachine::
+BufferingState::Step()
+{
+ TimeStamp now = TimeStamp::Now();
+ MOZ_ASSERT(!mBufferingStart.IsNull(), "Must know buffering start time.");
+
+ // With buffering heuristics we will remain in the buffering state if
+ // we've not decoded enough data to begin playback, or if we've not
+ // downloaded a reasonable amount of data inside our buffering time.
+ if (Reader()->UseBufferingHeuristics()) {
+ TimeDuration elapsed = now - mBufferingStart;
+ bool isLiveStream = Resource()->IsLiveStream();
+ if ((isLiveStream || !mMaster->CanPlayThrough()) &&
+ elapsed < TimeDuration::FromSeconds(mBufferingWait * mMaster->mPlaybackRate) &&
+ mMaster->HasLowBufferedData(mBufferingWait * USECS_PER_S) &&
+ IsExpectingMoreData()) {
+ SLOG("Buffering: wait %ds, timeout in %.3lfs",
+ mBufferingWait, mBufferingWait - elapsed.ToSeconds());
+ mMaster->ScheduleStateMachineIn(USECS_PER_S);
+ return;
+ }
+ } else if (mMaster->OutOfDecodedAudio() || mMaster->OutOfDecodedVideo()) {
+ MOZ_ASSERT(Reader()->IsWaitForDataSupported(),
+ "Don't yet have a strategy for non-heuristic + non-WaitForData");
+ mMaster->DispatchDecodeTasksIfNeeded();
+ MOZ_ASSERT(mMaster->mMinimizePreroll ||
+ !mMaster->OutOfDecodedAudio() ||
+ Reader()->IsRequestingAudioData() ||
+ Reader()->IsWaitingAudioData());
+ MOZ_ASSERT(mMaster->mMinimizePreroll ||
+ !mMaster->OutOfDecodedVideo() ||
+ Reader()->IsRequestingVideoData() ||
+ Reader()->IsWaitingVideoData());
+ SLOG("In buffering mode, waiting to be notified: outOfAudio: %d, "
+ "mAudioStatus: %s, outOfVideo: %d, mVideoStatus: %s",
+ mMaster->OutOfDecodedAudio(), mMaster->AudioRequestStatus(),
+ mMaster->OutOfDecodedVideo(), mMaster->VideoRequestStatus());
+ return;
+ }
+
+ SLOG("Buffered for %.3lfs", (now - mBufferingStart).ToSeconds());
+ SetState<DecodingState>();
+}
+
+void
+MediaDecoderStateMachine::
+BufferingState::HandleEndOfStream()
+{
+ if (mMaster->CheckIfDecodeComplete()) {
+ SetState<CompletedState>();
+ } else {
+ // Check if we can exit buffering.
+ mMaster->ScheduleStateMachine();
+ }
+}
+
+RefPtr<ShutdownPromise>
+MediaDecoderStateMachine::
+ShutdownState::Enter()
+{
+ auto master = mMaster;
+
+ master->mIsShutdown = true;
+ master->mDelayedScheduler.Reset();
+ master->mBufferedUpdateRequest.DisconnectIfExists();
+
+ // Shutdown happens while decode timer is active, we need to disconnect and
+ // dispose of the timer.
+ master->mVideoDecodeSuspendTimer.Reset();
+
+ master->mCDMProxyPromise.DisconnectIfExists();
+
+ if (master->IsPlaying()) {
+ master->StopPlayback();
+ }
+
+ // To break the cycle-reference between MediaDecoderReaderWrapper and MDSM.
+ master->CancelMediaDecoderReaderWrapperCallback();
+
+ master->Reset();
+
+ master->mMediaSink->Shutdown();
+
+ // Prevent dangling pointers by disconnecting the listeners.
+ master->mAudioQueueListener.Disconnect();
+ master->mVideoQueueListener.Disconnect();
+ master->mMetadataManager.Disconnect();
+ master->mOnMediaNotSeekable.Disconnect();
+
+ // Disconnect canonicals and mirrors before shutting down our task queue.
+ master->mBuffered.DisconnectIfConnected();
+ master->mEstimatedDuration.DisconnectIfConnected();
+ master->mExplicitDuration.DisconnectIfConnected();
+ master->mPlayState.DisconnectIfConnected();
+ master->mNextPlayState.DisconnectIfConnected();
+ master->mVolume.DisconnectIfConnected();
+ master->mPreservesPitch.DisconnectIfConnected();
+ master->mSameOriginMedia.DisconnectIfConnected();
+ master->mMediaPrincipalHandle.DisconnectIfConnected();
+ master->mPlaybackBytesPerSecond.DisconnectIfConnected();
+ master->mPlaybackRateReliable.DisconnectIfConnected();
+ master->mDecoderPosition.DisconnectIfConnected();
+ master->mIsVisible.DisconnectIfConnected();
+
+ master->mDuration.DisconnectAll();
+ master->mIsShutdown.DisconnectAll();
+ master->mNextFrameStatus.DisconnectAll();
+ master->mCurrentPosition.DisconnectAll();
+ master->mPlaybackOffset.DisconnectAll();
+ master->mIsAudioDataAudible.DisconnectAll();
+
+ // Shut down the watch manager to stop further notifications.
+ master->mWatchManager.Shutdown();
+
+ return Reader()->Shutdown()
+ ->Then(OwnerThread(), __func__, master,
+ &MediaDecoderStateMachine::FinishShutdown,
+ &MediaDecoderStateMachine::FinishShutdown)
+ ->CompletionPromise();
+}
+
+#define INIT_WATCHABLE(name, val) \
+ name(val, "MediaDecoderStateMachine::" #name)
+#define INIT_MIRROR(name, val) \
+ name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Mirror)")
+#define INIT_CANONICAL(name, val) \
+ name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Canonical)")
+
+MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
+ MediaDecoderReader* aReader) :
+ mDecoderID(aDecoder),
+ mFrameStats(&aDecoder->GetFrameStatistics()),
+ mVideoFrameContainer(aDecoder->GetVideoFrameContainer()),
+ mAudioChannel(aDecoder->GetAudioChannel()),
+ mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
+ /* aSupportsTailDispatch = */ true)),
+ mWatchManager(this, mTaskQueue),
+ mDispatchedStateMachine(false),
+ mDelayedScheduler(mTaskQueue),
+ mCurrentFrameID(0),
+ INIT_WATCHABLE(mObservedDuration, TimeUnit()),
+ mFragmentEndTime(-1),
+ mReader(new MediaDecoderReaderWrapper(mTaskQueue, aReader)),
+ mDecodedAudioEndTime(0),
+ mDecodedVideoEndTime(0),
+ mPlaybackRate(1.0),
+ mLowAudioThresholdUsecs(detail::LOW_AUDIO_USECS),
+ mAmpleAudioThresholdUsecs(detail::AMPLE_AUDIO_USECS),
+ mAudioCaptured(false),
+ mNotifyMetadataBeforeFirstFrame(false),
+ mMinimizePreroll(false),
+ mSentLoadedMetadataEvent(false),
+ mSentFirstFrameLoadedEvent(false),
+ mVideoDecodeSuspended(false),
+ mVideoDecodeSuspendTimer(mTaskQueue),
+ mOutputStreamManager(new OutputStreamManager()),
+ mResource(aDecoder->GetResource()),
+ mAudioOffloading(false),
+ INIT_MIRROR(mBuffered, TimeIntervals()),
+ INIT_MIRROR(mEstimatedDuration, NullableTimeUnit()),
+ INIT_MIRROR(mExplicitDuration, Maybe<double>()),
+ INIT_MIRROR(mPlayState, MediaDecoder::PLAY_STATE_LOADING),
+ INIT_MIRROR(mNextPlayState, MediaDecoder::PLAY_STATE_PAUSED),
+ INIT_MIRROR(mVolume, 1.0),
+ INIT_MIRROR(mPreservesPitch, true),
+ INIT_MIRROR(mSameOriginMedia, false),
+ INIT_MIRROR(mMediaPrincipalHandle, PRINCIPAL_HANDLE_NONE),
+ INIT_MIRROR(mPlaybackBytesPerSecond, 0.0),
+ INIT_MIRROR(mPlaybackRateReliable, true),
+ INIT_MIRROR(mDecoderPosition, 0),
+ INIT_MIRROR(mIsVisible, true),
+ INIT_CANONICAL(mDuration, NullableTimeUnit()),
+ INIT_CANONICAL(mIsShutdown, false),
+ INIT_CANONICAL(mNextFrameStatus, MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE),
+ INIT_CANONICAL(mCurrentPosition, 0),
+ INIT_CANONICAL(mPlaybackOffset, 0),
+ INIT_CANONICAL(mIsAudioDataAudible, false)
+{
+ MOZ_COUNT_CTOR(MediaDecoderStateMachine);
+ NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+
+ InitVideoQueuePrefs();
+
+#ifdef XP_WIN
+ // Ensure high precision timers are enabled on Windows, otherwise the state
+ // machine isn't woken up at reliable intervals to set the next frame,
+ // and we drop frames while painting. Note that multiple calls to this
+ // function per-process is OK, provided each call is matched by a corresponding
+ // timeEndPeriod() call.
+ timeBeginPeriod(1);
+#endif
+}
+
+#undef INIT_WATCHABLE
+#undef INIT_MIRROR
+#undef INIT_CANONICAL
+
+MediaDecoderStateMachine::~MediaDecoderStateMachine()
+{
+ MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread.");
+ MOZ_COUNT_DTOR(MediaDecoderStateMachine);
+
+#ifdef XP_WIN
+ timeEndPeriod(1);
+#endif
+}
+
+void
+MediaDecoderStateMachine::InitializationTask(MediaDecoder* aDecoder)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ // Connect mirrors.
+ mBuffered.Connect(mReader->CanonicalBuffered());
+ mEstimatedDuration.Connect(aDecoder->CanonicalEstimatedDuration());
+ mExplicitDuration.Connect(aDecoder->CanonicalExplicitDuration());
+ mPlayState.Connect(aDecoder->CanonicalPlayState());
+ mNextPlayState.Connect(aDecoder->CanonicalNextPlayState());
+ mVolume.Connect(aDecoder->CanonicalVolume());
+ mPreservesPitch.Connect(aDecoder->CanonicalPreservesPitch());
+ mSameOriginMedia.Connect(aDecoder->CanonicalSameOriginMedia());
+ mMediaPrincipalHandle.Connect(aDecoder->CanonicalMediaPrincipalHandle());
+ mPlaybackBytesPerSecond.Connect(aDecoder->CanonicalPlaybackBytesPerSecond());
+ mPlaybackRateReliable.Connect(aDecoder->CanonicalPlaybackRateReliable());
+ mDecoderPosition.Connect(aDecoder->CanonicalDecoderPosition());
+
+ // Initialize watchers.
+ mWatchManager.Watch(mBuffered, &MediaDecoderStateMachine::BufferedRangeUpdated);
+ mWatchManager.Watch(mVolume, &MediaDecoderStateMachine::VolumeChanged);
+ mWatchManager.Watch(mPreservesPitch, &MediaDecoderStateMachine::PreservesPitchChanged);
+ mWatchManager.Watch(mEstimatedDuration, &MediaDecoderStateMachine::RecomputeDuration);
+ mWatchManager.Watch(mExplicitDuration, &MediaDecoderStateMachine::RecomputeDuration);
+ mWatchManager.Watch(mObservedDuration, &MediaDecoderStateMachine::RecomputeDuration);
+ mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged);
+
+ if (MediaPrefs::MDSMSuspendBackgroundVideoEnabled()) {
+ mIsVisible.Connect(aDecoder->CanonicalIsVisible());
+ mWatchManager.Watch(mIsVisible, &MediaDecoderStateMachine::VisibilityChanged);
+ }
+
+ // Configure MediaDecoderReaderWrapper.
+ SetMediaDecoderReaderWrapperCallback();
+}
+
+void
+MediaDecoderStateMachine::AudioAudibleChanged(bool aAudible)
+{
+ mIsAudioDataAudible = aAudible;
+}
+
+media::MediaSink*
+MediaDecoderStateMachine::CreateAudioSink()
+{
+ RefPtr<MediaDecoderStateMachine> self = this;
+ auto audioSinkCreator = [self] () {
+ MOZ_ASSERT(self->OnTaskQueue());
+ DecodedAudioDataSink* audioSink = new DecodedAudioDataSink(
+ self->mTaskQueue, self->mAudioQueue, self->GetMediaTime(),
+ self->Info().mAudio, self->mAudioChannel);
+
+ self->mAudibleListener = audioSink->AudibleEvent().Connect(
+ self->mTaskQueue, self.get(), &MediaDecoderStateMachine::AudioAudibleChanged);
+ return audioSink;
+ };
+ return new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
+}
+
+already_AddRefed<media::MediaSink>
+MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured)
+{
+ RefPtr<media::MediaSink> audioSink = aAudioCaptured
+ ? new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue,
+ mOutputStreamManager, mSameOriginMedia.Ref(),
+ mMediaPrincipalHandle.Ref())
+ : CreateAudioSink();
+
+ RefPtr<media::MediaSink> mediaSink =
+ new VideoSink(mTaskQueue, audioSink, mVideoQueue,
+ mVideoFrameContainer, *mFrameStats,
+ sVideoQueueSendToCompositorSize);
+ return mediaSink.forget();
+}
+
+int64_t
+MediaDecoderStateMachine::GetDecodedAudioDuration()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mMediaSink->IsStarted()) {
+ // mDecodedAudioEndTime might be smaller than GetClock() when there is
+ // overlap between 2 adjacent audio samples or when we are playing
+ // a chained ogg file.
+ return std::max<int64_t>(mDecodedAudioEndTime - GetClock(), 0);
+ }
+ // MediaSink not started. All audio samples are in the queue.
+ return AudioQueue().Duration();
+}
+
+bool MediaDecoderStateMachine::HaveEnoughDecodedAudio()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ int64_t ampleAudioUSecs = mAmpleAudioThresholdUsecs * mPlaybackRate;
+ if (AudioQueue().GetSize() == 0 ||
+ GetDecodedAudioDuration() < ampleAudioUSecs) {
+ return false;
+ }
+
+ // MDSM will ensure buffering level is high enough for playback speed at 1x
+ // at which the DecodedStream is playing.
+ return true;
+}
+
+bool MediaDecoderStateMachine::HaveEnoughDecodedVideo()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (VideoQueue().GetSize() == 0) {
+ return false;
+ }
+
+ if (VideoQueue().GetSize() - 1 < GetAmpleVideoFrames() * mPlaybackRate) {
+ return false;
+ }
+
+ return true;
+}
+
+bool
+MediaDecoderStateMachine::NeedToDecodeVideo()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ SAMPLE_LOG("NeedToDecodeVideo() isDec=%d minPrl=%d enufVid=%d",
+ IsVideoDecoding(), mMinimizePreroll, HaveEnoughDecodedVideo());
+ return IsVideoDecoding() &&
+ mState != DECODER_STATE_SEEKING &&
+ ((!mSentFirstFrameLoadedEvent && VideoQueue().GetSize() == 0) ||
+ (!mMinimizePreroll && !HaveEnoughDecodedVideo()));
+}
+
+bool
+MediaDecoderStateMachine::NeedToSkipToNextKeyframe()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ // Don't skip when we're still decoding first frames.
+ if (!mSentFirstFrameLoadedEvent) {
+ return false;
+ }
+ MOZ_ASSERT(mState == DECODER_STATE_DECODING ||
+ mState == DECODER_STATE_BUFFERING ||
+ mState == DECODER_STATE_SEEKING);
+
+ // Since GetClock() can only be called after starting MediaSink, we return
+ // false quickly if it is not started because we won't fall behind playback
+ // when not consuming media data.
+ if (!mMediaSink->IsStarted()) {
+ return false;
+ }
+
+ // We are in seeking or buffering states, don't skip frame.
+ if (!IsVideoDecoding() || mState == DECODER_STATE_BUFFERING ||
+ mState == DECODER_STATE_SEEKING) {
+ return false;
+ }
+
+ // Don't skip frame for video-only decoded stream because the clock time of
+ // the stream relies on the video frame.
+ if (mAudioCaptured && !HasAudio()) {
+ return false;
+ }
+
+ // We'll skip the video decode to the next keyframe if we're low on
+ // audio, or if we're low on video, provided we're not running low on
+ // data to decode. If we're running low on downloaded data to decode,
+ // we won't start keyframe skipping, as we'll be pausing playback to buffer
+ // soon anyway and we'll want to be able to display frames immediately
+ // after buffering finishes. We ignore the low audio calculations for
+ // readers that are async, as since their audio decode runs on a different
+ // task queue it should never run low and skipping won't help their decode.
+ bool isLowOnDecodedAudio = !mReader->IsAsync() &&
+ IsAudioDecoding() &&
+ (GetDecodedAudioDuration() <
+ mLowAudioThresholdUsecs * mPlaybackRate);
+ bool isLowOnDecodedVideo = (GetClock() - mDecodedVideoEndTime) * mPlaybackRate >
+ LOW_VIDEO_THRESHOLD_USECS;
+ bool lowBuffered = HasLowBufferedData();
+
+ if ((isLowOnDecodedAudio || isLowOnDecodedVideo) && !lowBuffered) {
+ DECODER_LOG("Skipping video decode to the next keyframe lowAudio=%d lowVideo=%d lowUndecoded=%d async=%d",
+ isLowOnDecodedAudio, isLowOnDecodedVideo, lowBuffered, mReader->IsAsync());
+ return true;
+ }
+
+ return false;
+}
+
+bool
+MediaDecoderStateMachine::NeedToDecodeAudio()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ SAMPLE_LOG("NeedToDecodeAudio() isDec=%d minPrl=%d enufAud=%d",
+ IsAudioDecoding(), mMinimizePreroll, HaveEnoughDecodedAudio());
+
+ return IsAudioDecoding() &&
+ mState != DECODER_STATE_SEEKING &&
+ ((!mSentFirstFrameLoadedEvent && AudioQueue().GetSize() == 0) ||
+ (!mMinimizePreroll && !HaveEnoughDecodedAudio()));
+}
+
+void
+MediaDecoderStateMachine::OnAudioDecoded(MediaData* aAudio)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(aAudio);
+
+ // audio->GetEndTime() is not always mono-increasing in chained ogg.
+ mDecodedAudioEndTime = std::max(aAudio->GetEndTime(), mDecodedAudioEndTime);
+
+ SAMPLE_LOG("OnAudioDecoded [%lld,%lld]", aAudio->mTime, aAudio->GetEndTime());
+
+ mStateObj->HandleAudioDecoded(aAudio);
+}
+
+void
+MediaDecoderStateMachine::Push(MediaData* aSample, MediaData::Type aSampleType)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(aSample);
+
+ if (aSample->mType == MediaData::AUDIO_DATA) {
+ // TODO: Send aSample to MSG and recalculate readystate before pushing,
+ // otherwise AdvanceFrame may pop the sample before we have a chance
+ // to reach playing.
+ AudioQueue().Push(aSample);
+ } else if (aSample->mType == MediaData::VIDEO_DATA) {
+ // TODO: Send aSample to MSG and recalculate readystate before pushing,
+ // otherwise AdvanceFrame may pop the sample before we have a chance
+ // to reach playing.
+ aSample->As<VideoData>()->mFrameID = ++mCurrentFrameID;
+ VideoQueue().Push(aSample);
+ } else {
+ // TODO: Handle MediaRawData, determine which queue should be pushed.
+ }
+ DispatchDecodeTasksIfNeeded();
+}
+
+void
+MediaDecoderStateMachine::OnAudioPopped(const RefPtr<MediaData>& aSample)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
+ DispatchAudioDecodeTaskIfNeeded();
+}
+
+void
+MediaDecoderStateMachine::OnVideoPopped(const RefPtr<MediaData>& aSample)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
+ DispatchVideoDecodeTaskIfNeeded();
+}
+
+void
+MediaDecoderStateMachine::OnNotDecoded(MediaData::Type aType,
+ const MediaResult& aError)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState != DECODER_STATE_SEEKING);
+
+ SAMPLE_LOG("OnNotDecoded (aType=%u, aError=%u)", aType, aError.Code());
+ bool isAudio = aType == MediaData::AUDIO_DATA;
+ MOZ_ASSERT_IF(!isAudio, aType == MediaData::VIDEO_DATA);
+
+ if (IsShutdown()) {
+ // Already shutdown;
+ return;
+ }
+
+ // If the decoder is waiting for data, we tell it to call us back when the
+ // data arrives.
+ if (aError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
+ MOZ_ASSERT(mReader->IsWaitForDataSupported(),
+ "Readers that send WAITING_FOR_DATA need to implement WaitForData");
+ mReader->WaitForData(aType);
+ mStateObj->HandleWaitingForData();
+ return;
+ }
+
+ if (aError == NS_ERROR_DOM_MEDIA_CANCELED) {
+ if (isAudio) {
+ EnsureAudioDecodeTaskQueued();
+ } else {
+ EnsureVideoDecodeTaskQueued();
+ }
+ return;
+ }
+
+ // If this is a decode error, delegate to the generic error path.
+ if (aError != NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
+ DecodeError(aError);
+ return;
+ }
+
+ // This is an EOS. Finish off the queue, and then handle things based on our
+ // state.
+ if (isAudio) {
+ AudioQueue().Finish();
+ } else {
+ VideoQueue().Finish();
+ }
+
+ mStateObj->HandleEndOfStream();
+}
+
+void
+MediaDecoderStateMachine::OnVideoDecoded(MediaData* aVideo,
+ TimeStamp aDecodeStartTime)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(aVideo);
+
+ // Handle abnormal or negative timestamps.
+ mDecodedVideoEndTime = std::max(mDecodedVideoEndTime, aVideo->GetEndTime());
+
+ SAMPLE_LOG("OnVideoDecoded [%lld,%lld]", aVideo->mTime, aVideo->GetEndTime());
+
+ mStateObj->HandleVideoDecoded(aVideo, aDecodeStartTime);
+}
+
+bool
+MediaDecoderStateMachine::IsAudioDecoding()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return HasAudio() && !AudioQueue().IsFinished();
+}
+
+bool
+MediaDecoderStateMachine::IsVideoDecoding()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return HasVideo() && !VideoQueue().IsFinished();
+}
+
+bool
+MediaDecoderStateMachine::CheckIfDecodeComplete()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ // DecodeComplete is possible only after decoding first frames.
+ MOZ_ASSERT(mSentFirstFrameLoadedEvent);
+ MOZ_ASSERT(mState == DECODER_STATE_DECODING ||
+ mState == DECODER_STATE_BUFFERING);
+ return !IsVideoDecoding() && !IsAudioDecoding();
+}
+
+bool MediaDecoderStateMachine::IsPlaying() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return mMediaSink->IsPlaying();
+}
+
+void MediaDecoderStateMachine::SetMediaNotSeekable()
+{
+ mMediaSeekable = false;
+}
+
+void
+MediaDecoderStateMachine::OnAudioCallback(AudioCallbackData aData)
+{
+ if (aData.is<MediaData*>()) {
+ OnAudioDecoded(aData.as<MediaData*>());
+ } else {
+ OnNotDecoded(MediaData::AUDIO_DATA, aData.as<MediaResult>());
+ }
+}
+
+void
+MediaDecoderStateMachine::OnVideoCallback(VideoCallbackData aData)
+{
+ typedef Tuple<MediaData*, TimeStamp> Type;
+ if (aData.is<Type>()) {
+ auto&& v = aData.as<Type>();
+ OnVideoDecoded(Get<0>(v), Get<1>(v));
+ } else {
+ OnNotDecoded(MediaData::VIDEO_DATA, aData.as<MediaResult>());
+ }
+}
+
+void
+MediaDecoderStateMachine::OnAudioWaitCallback(WaitCallbackData aData)
+{
+ if (aData.is<MediaData::Type>()) {
+ EnsureAudioDecodeTaskQueued();
+ }
+}
+
+void
+MediaDecoderStateMachine::OnVideoWaitCallback(WaitCallbackData aData)
+{
+ if (aData.is<MediaData::Type>()) {
+ EnsureVideoDecodeTaskQueued();
+ }
+}
+
+nsresult MediaDecoderStateMachine::Init(MediaDecoder* aDecoder)
+{
+ MOZ_ASSERT(NS_IsMainThread());
+
+ // Dispatch initialization that needs to happen on that task queue.
+ nsCOMPtr<nsIRunnable> r = NewRunnableMethod<RefPtr<MediaDecoder>>(
+ this, &MediaDecoderStateMachine::InitializationTask, aDecoder);
+ mTaskQueue->Dispatch(r.forget());
+
+ mAudioQueueListener = AudioQueue().PopEvent().Connect(
+ mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped);
+ mVideoQueueListener = VideoQueue().PopEvent().Connect(
+ mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped);
+
+ mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
+
+ mOnMediaNotSeekable = mReader->OnMediaNotSeekable().Connect(
+ OwnerThread(), this, &MediaDecoderStateMachine::SetMediaNotSeekable);
+
+ mMediaSink = CreateMediaSink(mAudioCaptured);
+
+ mCDMProxyPromise.Begin(aDecoder->RequestCDMProxy()->Then(
+ OwnerThread(), __func__, this,
+ &MediaDecoderStateMachine::OnCDMProxyReady,
+ &MediaDecoderStateMachine::OnCDMProxyNotReady));
+
+ nsresult rv = mReader->Init();
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ RefPtr<MediaDecoderStateMachine> self = this;
+ OwnerThread()->Dispatch(NS_NewRunnableFunction([self] () {
+ MOZ_ASSERT(self->mState == DECODER_STATE_DECODING_METADATA);
+ MOZ_ASSERT(!self->mStateObj);
+ auto s = new DecodeMetadataState(self);
+ self->mStateObj.reset(s);
+ s->Enter();
+ }));
+
+ return NS_OK;
+}
+
+void
+MediaDecoderStateMachine::SetMediaDecoderReaderWrapperCallback()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ // The use of raw pointer references is safe because the lifecycle of a
+ // MediaDecoderStateMachine guarantees that the callbacks are disconnected
+ // before the MediaDecoderStateMachine is destroyed.
+ mAudioCallback = mReader->AudioCallback().Connect(
+ mTaskQueue, this, &MediaDecoderStateMachine::OnAudioCallback);
+
+ mVideoCallback = mReader->VideoCallback().Connect(
+ mTaskQueue, this, &MediaDecoderStateMachine::OnVideoCallback);
+
+ mAudioWaitCallback = mReader->AudioWaitCallback().Connect(
+ mTaskQueue, this, &MediaDecoderStateMachine::OnAudioWaitCallback);
+
+ mVideoWaitCallback = mReader->VideoWaitCallback().Connect(
+ mTaskQueue, this, &MediaDecoderStateMachine::OnVideoWaitCallback);
+}
+
+void
+MediaDecoderStateMachine::CancelMediaDecoderReaderWrapperCallback()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mAudioCallback.Disconnect();
+ mVideoCallback.Disconnect();
+ mAudioWaitCallback.Disconnect();
+ mVideoWaitCallback.Disconnect();
+}
+
+void MediaDecoderStateMachine::StopPlayback()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ DECODER_LOG("StopPlayback()");
+
+ mOnPlaybackEvent.Notify(MediaEventType::PlaybackStopped);
+
+ if (IsPlaying()) {
+ mMediaSink->SetPlaying(false);
+ MOZ_ASSERT(!IsPlaying());
+ }
+
+ DispatchDecodeTasksIfNeeded();
+}
+
+void MediaDecoderStateMachine::MaybeStartPlayback()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ // Should try to start playback only after decoding first frames.
+ MOZ_ASSERT(mSentFirstFrameLoadedEvent);
+ MOZ_ASSERT(mState == DECODER_STATE_DECODING ||
+ mState == DECODER_STATE_COMPLETED);
+
+ if (IsPlaying()) {
+ // Logging this case is really spammy - don't do it.
+ return;
+ }
+
+ bool playStatePermits = mPlayState == MediaDecoder::PLAY_STATE_PLAYING;
+ if (!playStatePermits || mAudioOffloading) {
+ DECODER_LOG("Not starting playback [playStatePermits: %d, "
+ "mAudioOffloading: %d]",
+ playStatePermits, mAudioOffloading);
+ return;
+ }
+
+ DECODER_LOG("MaybeStartPlayback() starting playback");
+ mOnPlaybackEvent.Notify(MediaEventType::PlaybackStarted);
+ StartMediaSink();
+
+ if (!IsPlaying()) {
+ mMediaSink->SetPlaying(true);
+ MOZ_ASSERT(IsPlaying());
+ }
+
+ DispatchDecodeTasksIfNeeded();
+}
+
+void MediaDecoderStateMachine::UpdatePlaybackPositionInternal(int64_t aTime)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ SAMPLE_LOG("UpdatePlaybackPositionInternal(%lld)", aTime);
+
+ mCurrentPosition = aTime;
+ NS_ASSERTION(mCurrentPosition >= 0, "CurrentTime should be positive!");
+ mObservedDuration = std::max(mObservedDuration.Ref(),
+ TimeUnit::FromMicroseconds(mCurrentPosition.Ref()));
+}
+
+void MediaDecoderStateMachine::UpdatePlaybackPosition(int64_t aTime)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ UpdatePlaybackPositionInternal(aTime);
+
+ bool fragmentEnded = mFragmentEndTime >= 0 && GetMediaTime() >= mFragmentEndTime;
+ mMetadataManager.DispatchMetadataIfNeeded(TimeUnit::FromMicroseconds(aTime));
+
+ if (fragmentEnded) {
+ StopPlayback();
+ }
+}
+
+/* static */ const char*
+MediaDecoderStateMachine::ToStateStr(State aState)
+{
+ switch (aState) {
+ case DECODER_STATE_DECODING_METADATA: return "DECODING_METADATA";
+ case DECODER_STATE_WAIT_FOR_CDM: return "WAIT_FOR_CDM";
+ case DECODER_STATE_DORMANT: return "DORMANT";
+ case DECODER_STATE_DECODING_FIRSTFRAME: return "DECODING_FIRSTFRAME";
+ case DECODER_STATE_DECODING: return "DECODING";
+ case DECODER_STATE_SEEKING: return "SEEKING";
+ case DECODER_STATE_BUFFERING: return "BUFFERING";
+ case DECODER_STATE_COMPLETED: return "COMPLETED";
+ case DECODER_STATE_SHUTDOWN: return "SHUTDOWN";
+ default: MOZ_ASSERT_UNREACHABLE("Invalid state.");
+ }
+ return "UNKNOWN";
+}
+
+const char*
+MediaDecoderStateMachine::ToStateStr()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return ToStateStr(mState);
+}
+
+void MediaDecoderStateMachine::VolumeChanged()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mMediaSink->SetVolume(mVolume);
+}
+
+void MediaDecoderStateMachine::RecomputeDuration()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ TimeUnit duration;
+ if (mExplicitDuration.Ref().isSome()) {
+ double d = mExplicitDuration.Ref().ref();
+ if (IsNaN(d)) {
+ // We have an explicit duration (which means that we shouldn't look at
+ // any other duration sources), but the duration isn't ready yet.
+ return;
+ }
+ // We don't fire duration changed for this case because it should have
+ // already been fired on the main thread when the explicit duration was set.
+ duration = TimeUnit::FromSeconds(d);
+ } else if (mEstimatedDuration.Ref().isSome()) {
+ duration = mEstimatedDuration.Ref().ref();
+ } else if (Info().mMetadataDuration.isSome()) {
+ duration = Info().mMetadataDuration.ref();
+ } else {
+ return;
+ }
+
+ // Only adjust the duration when an explicit duration isn't set (MSE).
+ // The duration is always exactly known with MSE and there's no need to adjust
+ // it based on what may have been seen in the past; in particular as this data
+ // may no longer exist such as when the mediasource duration was reduced.
+ if (mExplicitDuration.Ref().isNothing() &&
+ duration < mObservedDuration.Ref()) {
+ duration = mObservedDuration;
+ }
+
+ MOZ_ASSERT(duration.ToMicroseconds() >= 0);
+ mDuration = Some(duration);
+}
+
+RefPtr<ShutdownPromise>
+MediaDecoderStateMachine::Shutdown()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return mStateObj->HandleShutdown();
+}
+
+void MediaDecoderStateMachine::PlayStateChanged()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) {
+ mVideoDecodeSuspendTimer.Reset();
+ } else if (mMinimizePreroll) {
+ // Once we start playing, we don't want to minimize our prerolling, as we
+ // assume the user is likely to want to keep playing in future. This needs to
+ // happen before we invoke StartDecoding().
+ mMinimizePreroll = false;
+ DispatchDecodeTasksIfNeeded();
+ }
+
+ mStateObj->HandlePlayStateChanged(mPlayState);
+}
+
+void MediaDecoderStateMachine::VisibilityChanged()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ DECODER_LOG("VisibilityChanged: mIsVisible=%d, mVideoDecodeSuspended=%c",
+ mIsVisible.Ref(), mVideoDecodeSuspended ? 'T' : 'F');
+
+ // Start timer to trigger suspended decoding state when going invisible.
+ if (!mIsVisible) {
+ TimeStamp target = TimeStamp::Now() + SuspendBackgroundVideoDelay();
+
+ RefPtr<MediaDecoderStateMachine> self = this;
+ mVideoDecodeSuspendTimer.Ensure(target,
+ [=]() { self->OnSuspendTimerResolved(); },
+ [=]() { self->OnSuspendTimerRejected(); });
+ return;
+ }
+
+ // Resuming from suspended decoding
+
+ // If suspend timer exists, destroy it.
+ mVideoDecodeSuspendTimer.Reset();
+
+ if (mVideoDecodeSuspended) {
+ mStateObj->HandleResumeVideoDecoding();
+ }
+}
+
+void MediaDecoderStateMachine::BufferedRangeUpdated()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ // While playing an unseekable stream of unknown duration, mObservedDuration
+ // is updated (in AdvanceFrame()) as we play. But if data is being downloaded
+ // faster than played, mObserved won't reflect the end of playable data
+ // since we haven't played the frame at the end of buffered data. So update
+ // mObservedDuration here as new data is downloaded to prevent such a lag.
+ if (!mBuffered.Ref().IsInvalid()) {
+ bool exists;
+ media::TimeUnit end{mBuffered.Ref().GetEnd(&exists)};
+ if (exists) {
+ mObservedDuration = std::max(mObservedDuration.Ref(), end);
+ }
+ }
+}
+
+RefPtr<MediaDecoder::SeekPromise>
+MediaDecoderStateMachine::Seek(SeekTarget aTarget)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (IsShutdown()) {
+ return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__);
+ }
+
+ // We need to be able to seek in some way
+ if (!mMediaSeekable && !mMediaSeekableOnlyInBufferedRanges) {
+ DECODER_WARN("Seek() function should not be called on a non-seekable state machine");
+ return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__);
+ }
+
+ if (aTarget.IsNextFrame() && !HasVideo()) {
+ DECODER_WARN("Ignore a NextFrameSeekTask on a media file without video track.");
+ return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__);
+ }
+
+ MOZ_ASSERT(mDuration.Ref().isSome(), "We should have got duration already");
+
+ return mStateObj->HandleSeek(aTarget);
+}
+
+RefPtr<MediaDecoder::SeekPromise>
+MediaDecoderStateMachine::InvokeSeek(SeekTarget aTarget)
+{
+ return InvokeAsync(OwnerThread(), this, __func__,
+ &MediaDecoderStateMachine::Seek, aTarget);
+}
+
+void MediaDecoderStateMachine::StopMediaSink()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mMediaSink->IsStarted()) {
+ DECODER_LOG("Stop MediaSink");
+ mAudibleListener.DisconnectIfExists();
+
+ mMediaSink->Stop();
+ mMediaSinkAudioPromise.DisconnectIfExists();
+ mMediaSinkVideoPromise.DisconnectIfExists();
+ }
+}
+
+void
+MediaDecoderStateMachine::DispatchDecodeTasksIfNeeded()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (mState != DECODER_STATE_DECODING &&
+ mState != DECODER_STATE_DECODING_FIRSTFRAME &&
+ mState != DECODER_STATE_BUFFERING &&
+ mState != DECODER_STATE_SEEKING) {
+ return;
+ }
+
+ // NeedToDecodeAudio() can go from false to true while we hold the
+ // monitor, but it can't go from true to false. This can happen because
+ // NeedToDecodeAudio() takes into account the amount of decoded audio
+ // that's been written to the AudioStream but not played yet. So if we
+ // were calling NeedToDecodeAudio() twice and we thread-context switch
+ // between the calls, audio can play, which can affect the return value
+ // of NeedToDecodeAudio() giving inconsistent results. So we cache the
+ // value returned by NeedToDecodeAudio(), and make decisions
+ // based on the cached value. If NeedToDecodeAudio() has
+ // returned false, and then subsequently returns true and we're not
+ // playing, it will probably be OK since we don't need to consume data
+ // anyway.
+
+ const bool needToDecodeAudio = NeedToDecodeAudio();
+ const bool needToDecodeVideo = NeedToDecodeVideo();
+
+ // If we're in completed state, we should not need to decode anything else.
+ MOZ_ASSERT(mState != DECODER_STATE_COMPLETED ||
+ (!needToDecodeAudio && !needToDecodeVideo));
+
+ bool needIdle = !IsLogicallyPlaying() &&
+ mState != DECODER_STATE_SEEKING &&
+ !needToDecodeAudio &&
+ !needToDecodeVideo &&
+ !IsPlaying();
+
+ SAMPLE_LOG("DispatchDecodeTasksIfNeeded needAudio=%d audioStatus=%s needVideo=%d videoStatus=%s needIdle=%d",
+ needToDecodeAudio, AudioRequestStatus(),
+ needToDecodeVideo, VideoRequestStatus(),
+ needIdle);
+
+ if (needToDecodeAudio) {
+ EnsureAudioDecodeTaskQueued();
+ }
+ if (needToDecodeVideo) {
+ EnsureVideoDecodeTaskQueued();
+ }
+
+ if (needIdle) {
+ DECODER_LOG("Dispatching SetIdle() audioQueue=%lld videoQueue=%lld",
+ GetDecodedAudioDuration(),
+ VideoQueue().Duration());
+ mReader->SetIdle();
+ }
+}
+
+void
+MediaDecoderStateMachine::DispatchAudioDecodeTaskIfNeeded()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (!IsShutdown() && NeedToDecodeAudio()) {
+ EnsureAudioDecodeTaskQueued();
+ }
+}
+
+void
+MediaDecoderStateMachine::EnsureAudioDecodeTaskQueued()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState != DECODER_STATE_SEEKING);
+
+ SAMPLE_LOG("EnsureAudioDecodeTaskQueued isDecoding=%d status=%s",
+ IsAudioDecoding(), AudioRequestStatus());
+
+ if (mState != DECODER_STATE_DECODING &&
+ mState != DECODER_STATE_DECODING_FIRSTFRAME &&
+ mState != DECODER_STATE_BUFFERING) {
+ return;
+ }
+
+ if (!IsAudioDecoding() ||
+ mReader->IsRequestingAudioData() ||
+ mReader->IsWaitingAudioData()) {
+ return;
+ }
+
+ RequestAudioData();
+}
+
+void
+MediaDecoderStateMachine::RequestAudioData()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState != DECODER_STATE_SEEKING);
+
+ SAMPLE_LOG("Queueing audio task - queued=%i, decoder-queued=%o",
+ AudioQueue().GetSize(), mReader->SizeOfAudioQueueInFrames());
+
+ mReader->RequestAudioData();
+}
+
+void
+MediaDecoderStateMachine::DispatchVideoDecodeTaskIfNeeded()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (!IsShutdown() && NeedToDecodeVideo()) {
+ EnsureVideoDecodeTaskQueued();
+ }
+}
+
+void
+MediaDecoderStateMachine::EnsureVideoDecodeTaskQueued()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState != DECODER_STATE_SEEKING);
+
+ SAMPLE_LOG("EnsureVideoDecodeTaskQueued isDecoding=%d status=%s",
+ IsVideoDecoding(), VideoRequestStatus());
+
+ if (mState != DECODER_STATE_DECODING &&
+ mState != DECODER_STATE_DECODING_FIRSTFRAME &&
+ mState != DECODER_STATE_BUFFERING) {
+ return;
+ }
+
+ if (!IsVideoDecoding() ||
+ mReader->IsRequestingVideoData() ||
+ mReader->IsWaitingVideoData()) {
+ return;
+ }
+
+ RequestVideoData();
+}
+
+void
+MediaDecoderStateMachine::RequestVideoData()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState != DECODER_STATE_SEEKING);
+
+ bool skipToNextKeyFrame = NeedToSkipToNextKeyframe();
+
+ media::TimeUnit currentTime = media::TimeUnit::FromMicroseconds(GetMediaTime());
+
+ SAMPLE_LOG("Queueing video task - queued=%i, decoder-queued=%o, skip=%i, time=%lld",
+ VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), skipToNextKeyFrame,
+ currentTime.ToMicroseconds());
+
+ // MediaDecoderReaderWrapper::RequestVideoData() records the decoding start
+ // time and sent it back to MDSM::OnVideoDecoded() so that if the decoding is
+ // slow, we can increase our low audio threshold to reduce the chance of an
+ // audio underrun while we're waiting for a video decode to complete.
+ mReader->RequestVideoData(skipToNextKeyFrame, currentTime);
+}
+
+void
+MediaDecoderStateMachine::StartMediaSink()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (!mMediaSink->IsStarted()) {
+ mAudioCompleted = false;
+ mMediaSink->Start(GetMediaTime(), Info());
+
+ auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack);
+ auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
+
+ if (audioPromise) {
+ mMediaSinkAudioPromise.Begin(audioPromise->Then(
+ OwnerThread(), __func__, this,
+ &MediaDecoderStateMachine::OnMediaSinkAudioComplete,
+ &MediaDecoderStateMachine::OnMediaSinkAudioError));
+ }
+ if (videoPromise) {
+ mMediaSinkVideoPromise.Begin(videoPromise->Then(
+ OwnerThread(), __func__, this,
+ &MediaDecoderStateMachine::OnMediaSinkVideoComplete,
+ &MediaDecoderStateMachine::OnMediaSinkVideoError));
+ }
+ }
+}
+
+bool
+MediaDecoderStateMachine::HasLowDecodedAudio()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return IsAudioDecoding() &&
+ GetDecodedAudioDuration() < EXHAUSTED_DATA_MARGIN_USECS * mPlaybackRate;
+}
+
+bool
+MediaDecoderStateMachine::HasLowDecodedVideo()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return IsVideoDecoding() &&
+ VideoQueue().GetSize() < LOW_VIDEO_FRAMES * mPlaybackRate;
+}
+
+bool
+MediaDecoderStateMachine::HasLowDecodedData()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mReader->UseBufferingHeuristics());
+ return HasLowDecodedAudio() || HasLowDecodedVideo();
+}
+
+bool MediaDecoderStateMachine::OutOfDecodedAudio()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return IsAudioDecoding() && !AudioQueue().IsFinished() &&
+ AudioQueue().GetSize() == 0 &&
+ !mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
+}
+
+bool MediaDecoderStateMachine::HasLowBufferedData()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return HasLowBufferedData(detail::LOW_DATA_THRESHOLD_USECS);
+}
+
+bool MediaDecoderStateMachine::HasLowBufferedData(int64_t aUsecs)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState >= DECODER_STATE_DECODING,
+ "Must have loaded first frame for mBuffered to be valid");
+
+ // If we don't have a duration, mBuffered is probably not going to have
+ // a useful buffered range. Return false here so that we don't get stuck in
+ // buffering mode for live streams.
+ if (Duration().IsInfinite()) {
+ return false;
+ }
+
+ if (mBuffered.Ref().IsInvalid()) {
+ return false;
+ }
+
+ // We are never low in decoded data when we don't have audio/video or have
+ // decoded all audio/video samples.
+ int64_t endOfDecodedVideoData =
+ (HasVideo() && !VideoQueue().IsFinished())
+ ? mDecodedVideoEndTime
+ : INT64_MAX;
+ int64_t endOfDecodedAudioData =
+ (HasAudio() && !AudioQueue().IsFinished())
+ ? mDecodedAudioEndTime
+ : INT64_MAX;
+
+ int64_t endOfDecodedData = std::min(endOfDecodedVideoData, endOfDecodedAudioData);
+ if (Duration().ToMicroseconds() < endOfDecodedData) {
+ // Our duration is not up to date. No point buffering.
+ return false;
+ }
+
+ if (endOfDecodedData == INT64_MAX) {
+ // Have decoded all samples. No point buffering.
+ return false;
+ }
+
+ int64_t start = endOfDecodedData;
+ int64_t end = std::min(GetMediaTime() + aUsecs, Duration().ToMicroseconds());
+ if (start >= end) {
+ // Duration of decoded samples is greater than our threshold.
+ return false;
+ }
+ media::TimeInterval interval(media::TimeUnit::FromMicroseconds(start),
+ media::TimeUnit::FromMicroseconds(end));
+ return !mBuffered.Ref().Contains(interval);
+}
+
+void
+MediaDecoderStateMachine::DecodeError(const MediaResult& aError)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(!IsShutdown());
+ DECODER_WARN("Decode error");
+ // Notify the decode error and MediaDecoder will shut down MDSM.
+ mOnPlaybackErrorEvent.Notify(aError);
+}
+
+void
+MediaDecoderStateMachine::EnqueueLoadedMetadataEvent()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MediaDecoderEventVisibility visibility =
+ mSentLoadedMetadataEvent ? MediaDecoderEventVisibility::Suppressed
+ : MediaDecoderEventVisibility::Observable;
+ mMetadataLoadedEvent.Notify(nsAutoPtr<MediaInfo>(new MediaInfo(Info())),
+ Move(mMetadataTags),
+ visibility);
+ mSentLoadedMetadataEvent = true;
+}
+
+void
+MediaDecoderStateMachine::EnqueueFirstFrameLoadedEvent()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ // Track value of mSentFirstFrameLoadedEvent from before updating it
+ bool firstFrameBeenLoaded = mSentFirstFrameLoadedEvent;
+ mSentFirstFrameLoadedEvent = true;
+ RefPtr<MediaDecoderStateMachine> self = this;
+ mBufferedUpdateRequest.Begin(
+ mReader->UpdateBufferedWithPromise()
+ ->Then(OwnerThread(),
+ __func__,
+ // Resolve
+ [self, firstFrameBeenLoaded]() {
+ self->mBufferedUpdateRequest.Complete();
+ MediaDecoderEventVisibility visibility =
+ firstFrameBeenLoaded ? MediaDecoderEventVisibility::Suppressed
+ : MediaDecoderEventVisibility::Observable;
+ self->mFirstFrameLoadedEvent.Notify(
+ nsAutoPtr<MediaInfo>(new MediaInfo(self->Info())), visibility);
+ },
+ // Reject
+ []() { MOZ_CRASH("Should not reach"); }));
+}
+
+void
+MediaDecoderStateMachine::FinishDecodeFirstFrame()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(!mSentFirstFrameLoadedEvent);
+ DECODER_LOG("FinishDecodeFirstFrame");
+
+ mMediaSink->Redraw(Info().mVideo);
+
+ // If we don't know the duration by this point, we assume infinity, per spec.
+ if (mDuration.Ref().isNothing()) {
+ mDuration = Some(TimeUnit::FromInfinity());
+ }
+
+ DECODER_LOG("Media duration %lld, "
+ "transportSeekable=%d, mediaSeekable=%d",
+ Duration().ToMicroseconds(), mResource->IsTransportSeekable(), mMediaSeekable);
+
+ // Get potentially updated metadata
+ mReader->ReadUpdatedMetadata(mInfo.ptr());
+
+ if (!mNotifyMetadataBeforeFirstFrame) {
+ // If we didn't have duration and/or start time before, we should now.
+ EnqueueLoadedMetadataEvent();
+ }
+
+ EnqueueFirstFrameLoadedEvent();
+}
+
+RefPtr<ShutdownPromise>
+MediaDecoderStateMachine::BeginShutdown()
+{
+ return InvokeAsync(OwnerThread(), this, __func__,
+ &MediaDecoderStateMachine::Shutdown);
+}
+
+RefPtr<ShutdownPromise>
+MediaDecoderStateMachine::FinishShutdown()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(mState == DECODER_STATE_SHUTDOWN,
+ "How did we escape from the shutdown state?");
+ DECODER_LOG("Shutting down state machine task queue");
+ return OwnerThread()->BeginShutdown();
+}
+
+void
+MediaDecoderStateMachine::RunStateMachine()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ mDelayedScheduler.Reset(); // Must happen on state machine task queue.
+ mDispatchedStateMachine = false;
+ mStateObj->Step();
+}
+
+void
+MediaDecoderStateMachine::Reset(TrackSet aTracks)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ DECODER_LOG("MediaDecoderStateMachine::Reset");
+
+ // We should be resetting because we're seeking, shutting down, or entering
+ // dormant state. We could also be in the process of going dormant, and have
+ // just switched to exiting dormant before we finished entering dormant,
+ // hence the DECODING_NONE case below.
+ MOZ_ASSERT(IsShutdown() ||
+ mState == DECODER_STATE_SEEKING ||
+ mState == DECODER_STATE_DORMANT);
+
+ // Assert that aTracks specifies to reset the video track because we
+ // don't currently support resetting just the audio track.
+ MOZ_ASSERT(aTracks.contains(TrackInfo::kVideoTrack));
+
+ if (aTracks.contains(TrackInfo::kAudioTrack) &&
+ aTracks.contains(TrackInfo::kVideoTrack)) {
+ // Stop the audio thread. Otherwise, MediaSink might be accessing AudioQueue
+ // outside of the decoder monitor while we are clearing the queue and causes
+ // crash for no samples to be popped.
+ StopMediaSink();
+ }
+
+ if (aTracks.contains(TrackInfo::kVideoTrack)) {
+ mDecodedVideoEndTime = 0;
+ mVideoCompleted = false;
+ VideoQueue().Reset();
+ }
+
+ if (aTracks.contains(TrackInfo::kAudioTrack)) {
+ mDecodedAudioEndTime = 0;
+ mAudioCompleted = false;
+ AudioQueue().Reset();
+ }
+
+ mPlaybackOffset = 0;
+
+ mReader->ResetDecode(aTracks);
+}
+
+int64_t
+MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ int64_t clockTime = mMediaSink->GetPosition(aTimeStamp);
+ NS_ASSERTION(GetMediaTime() <= clockTime, "Clock should go forwards.");
+ return clockTime;
+}
+
+void
+MediaDecoderStateMachine::UpdatePlaybackPositionPeriodically()
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (!IsPlaying()) {
+ return;
+ }
+
+ // Cap the current time to the larger of the audio and video end time.
+ // This ensures that if we're running off the system clock, we don't
+ // advance the clock to after the media end time.
+ if (VideoEndTime() != -1 || AudioEndTime() != -1) {
+
+ const int64_t clockTime = GetClock();
+ // Skip frames up to the frame at the playback position, and figure out
+ // the time remaining until it's time to display the next frame and drop
+ // the current frame.
+ NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
+
+ // These will be non -1 if we've displayed a video frame, or played an audio frame.
+ int64_t t = std::min(clockTime, std::max(VideoEndTime(), AudioEndTime()));
+ // FIXME: Bug 1091422 - chained ogg files hit this assertion.
+ //MOZ_ASSERT(t >= GetMediaTime());
+ if (t > GetMediaTime()) {
+ UpdatePlaybackPosition(t);
+ }
+ }
+ // Note we have to update playback position before releasing the monitor.
+ // Otherwise, MediaDecoder::AddOutputStream could kick in when we are outside
+ // the monitor and get a staled value from GetCurrentTimeUs() which hits the
+ // assertion in GetClock().
+
+ int64_t delay = std::max<int64_t>(1, AUDIO_DURATION_USECS / mPlaybackRate);
+ ScheduleStateMachineIn(delay);
+}
+
+/* static */ const char*
+MediaDecoderStateMachine::ToStr(NextFrameStatus aStatus)
+{
+ switch (aStatus) {
+ case MediaDecoderOwner::NEXT_FRAME_AVAILABLE: return "NEXT_FRAME_AVAILABLE";
+ case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE: return "NEXT_FRAME_UNAVAILABLE";
+ case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING: return "NEXT_FRAME_UNAVAILABLE_BUFFERING";
+ case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING: return "NEXT_FRAME_UNAVAILABLE_SEEKING";
+ case MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED: return "NEXT_FRAME_UNINITIALIZED";
+ }
+ return "UNKNOWN";
+}
+
+void
+MediaDecoderStateMachine::UpdateNextFrameStatus(NextFrameStatus aStatus)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (aStatus != mNextFrameStatus) {
+ DECODER_LOG("Changed mNextFrameStatus to %s", ToStr(aStatus));
+ mNextFrameStatus = aStatus;
+ }
+}
+
+bool
+MediaDecoderStateMachine::CanPlayThrough()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return GetStatistics().CanPlayThrough();
+}
+
+MediaStatistics
+MediaDecoderStateMachine::GetStatistics()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MediaStatistics result;
+ result.mDownloadRate = mResource->GetDownloadRate(&result.mDownloadRateReliable);
+ result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition);
+ result.mTotalBytes = mResource->GetLength();
+ result.mPlaybackRate = mPlaybackBytesPerSecond;
+ result.mPlaybackRateReliable = mPlaybackRateReliable;
+ result.mDecoderPosition = mDecoderPosition;
+ result.mPlaybackPosition = mPlaybackOffset;
+ return result;
+}
+
+void
+MediaDecoderStateMachine::ScheduleStateMachine()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mDispatchedStateMachine) {
+ return;
+ }
+ mDispatchedStateMachine = true;
+
+ OwnerThread()->Dispatch(NewRunnableMethod(this, &MediaDecoderStateMachine::RunStateMachine));
+}
+
+void
+MediaDecoderStateMachine::ScheduleStateMachineIn(int64_t aMicroseconds)
+{
+ MOZ_ASSERT(OnTaskQueue()); // mDelayedScheduler.Ensure() may Disconnect()
+ // the promise, which must happen on the state
+ // machine task queue.
+ MOZ_ASSERT(aMicroseconds > 0);
+ if (mDispatchedStateMachine) {
+ return;
+ }
+
+ TimeStamp now = TimeStamp::Now();
+ TimeStamp target = now + TimeDuration::FromMicroseconds(aMicroseconds);
+
+ // It is OK to capture 'this' without causing UAF because the callback
+ // always happens before shutdown.
+ RefPtr<MediaDecoderStateMachine> self = this;
+ mDelayedScheduler.Ensure(target, [self] () {
+ self->mDelayedScheduler.CompleteRequest();
+ self->RunStateMachine();
+ }, [] () {
+ MOZ_DIAGNOSTIC_ASSERT(false);
+ });
+}
+
+bool MediaDecoderStateMachine::OnTaskQueue() const
+{
+ return OwnerThread()->IsCurrentThreadIn();
+}
+
+bool MediaDecoderStateMachine::IsStateMachineScheduled() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return mDispatchedStateMachine || mDelayedScheduler.IsScheduled();
+}
+
+void
+MediaDecoderStateMachine::SetPlaybackRate(double aPlaybackRate)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(aPlaybackRate != 0, "Should be handled by MediaDecoder::Pause()");
+
+ mPlaybackRate = aPlaybackRate;
+ mMediaSink->SetPlaybackRate(mPlaybackRate);
+
+ // Schedule next cycle to check if we can stop prerolling.
+ ScheduleStateMachine();
+}
+
+void MediaDecoderStateMachine::PreservesPitchChanged()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mMediaSink->SetPreservesPitch(mPreservesPitch);
+}
+
+bool
+MediaDecoderStateMachine::IsShutdown() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return mIsShutdown;
+}
+
+int64_t
+MediaDecoderStateMachine::AudioEndTime() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mMediaSink->IsStarted()) {
+ return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
+ }
+ MOZ_ASSERT(!HasAudio());
+ return -1;
+}
+
+int64_t
+MediaDecoderStateMachine::VideoEndTime() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mMediaSink->IsStarted()) {
+ return mMediaSink->GetEndTime(TrackInfo::kVideoTrack);
+ }
+ return -1;
+}
+
+void
+MediaDecoderStateMachine::OnMediaSinkVideoComplete()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(HasVideo());
+ VERBOSE_LOG("[%s]", __func__);
+
+ mMediaSinkVideoPromise.Complete();
+ mVideoCompleted = true;
+ ScheduleStateMachine();
+}
+
+void
+MediaDecoderStateMachine::OnMediaSinkVideoError()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(HasVideo());
+ VERBOSE_LOG("[%s]", __func__);
+
+ mMediaSinkVideoPromise.Complete();
+ mVideoCompleted = true;
+ if (HasAudio()) {
+ return;
+ }
+ DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__));
+}
+
+void MediaDecoderStateMachine::OnMediaSinkAudioComplete()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(HasAudio());
+ VERBOSE_LOG("[%s]", __func__);
+
+ mMediaSinkAudioPromise.Complete();
+ mAudioCompleted = true;
+ // To notify PlaybackEnded as soon as possible.
+ ScheduleStateMachine();
+
+ // Report OK to Decoder Doctor (to know if issue may have been resolved).
+ mOnDecoderDoctorEvent.Notify(
+ DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, NS_OK});
+}
+
+void MediaDecoderStateMachine::OnMediaSinkAudioError(nsresult aResult)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(HasAudio());
+ VERBOSE_LOG("[%s]", __func__);
+
+ mMediaSinkAudioPromise.Complete();
+ mAudioCompleted = true;
+
+ // Result should never be NS_OK in this *error* handler. Report to Dec-Doc.
+ MOZ_ASSERT(NS_FAILED(aResult));
+ mOnDecoderDoctorEvent.Notify(
+ DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, aResult});
+
+ // Make the best effort to continue playback when there is video.
+ if (HasVideo()) {
+ return;
+ }
+
+ // Otherwise notify media decoder/element about this error for it makes
+ // no sense to play an audio-only file without sound output.
+ DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__));
+}
+
+void
+MediaDecoderStateMachine::OnCDMProxyReady(RefPtr<CDMProxy> aProxy)
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mCDMProxyPromise.Complete();
+ mCDMProxy = aProxy;
+ mReader->SetCDMProxy(aProxy);
+ mStateObj->HandleCDMProxyReady();
+}
+
+void
+MediaDecoderStateMachine::OnCDMProxyNotReady()
+{
+ MOZ_ASSERT(OnTaskQueue());
+ mCDMProxyPromise.Complete();
+}
+
+void
+MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured)
+{
+ MOZ_ASSERT(OnTaskQueue());
+
+ if (aCaptured == mAudioCaptured) {
+ return;
+ }
+
+ // Rest these flags so they are consistent with the status of the sink.
+ // TODO: Move these flags into MediaSink to improve cohesion so we don't need
+ // to reset these flags when switching MediaSinks.
+ mAudioCompleted = false;
+ mVideoCompleted = false;
+
+ // Backup current playback parameters.
+ MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams();
+
+ // Stop and shut down the existing sink.
+ StopMediaSink();
+ mMediaSink->Shutdown();
+
+ // Create a new sink according to whether audio is captured.
+ mMediaSink = CreateMediaSink(aCaptured);
+
+ // Restore playback parameters.
+ mMediaSink->SetPlaybackParams(params);
+
+ mAudioCaptured = aCaptured;
+
+ // Don't buffer as much when audio is captured because we don't need to worry
+ // about high latency audio devices.
+ mAmpleAudioThresholdUsecs = mAudioCaptured ?
+ detail::AMPLE_AUDIO_USECS / 2 :
+ detail::AMPLE_AUDIO_USECS;
+
+ mStateObj->HandleAudioCaptured();
+}
+
+uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ return (mReader->IsAsync() && mReader->VideoIsHardwareAccelerated())
+ ? std::max<uint32_t>(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE)
+ : std::max<uint32_t>(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE);
+}
+
+void
+MediaDecoderStateMachine::DumpDebugInfo()
+{
+ MOZ_ASSERT(NS_IsMainThread());
+
+ // It is fine to capture a raw pointer here because MediaDecoder only call
+ // this function before shutdown begins.
+ RefPtr<MediaDecoderStateMachine> self = this;
+ nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([this, self] () {
+ mMediaSink->DumpDebugInfo();
+ mStateObj->DumpDebugInfo();
+ DUMP_LOG(
+ "GetMediaTime=%lld GetClock=%lld mMediaSink=%p "
+ "mState=%s mPlayState=%d mSentFirstFrameLoadedEvent=%d IsPlaying=%d "
+ "mAudioStatus=%s mVideoStatus=%s mDecodedAudioEndTime=%lld mDecodedVideoEndTime=%lld "
+ "mAudioCompleted=%d mVideoCompleted=%d",
+ GetMediaTime(), mMediaSink->IsStarted() ? GetClock() : -1, mMediaSink.get(),
+ ToStateStr(), mPlayState.Ref(), mSentFirstFrameLoadedEvent, IsPlaying(),
+ AudioRequestStatus(), VideoRequestStatus(), mDecodedAudioEndTime, mDecodedVideoEndTime,
+ mAudioCompleted, mVideoCompleted);
+ });
+
+ // Since the task is run asynchronously, it is possible other tasks get first
+ // and change the object states before we print them. Therefore we want to
+ // dispatch this task immediately without waiting for the tail dispatching
+ // phase so object states are less likely to change before being printed.
+ OwnerThread()->Dispatch(r.forget(),
+ AbstractThread::AssertDispatchSuccess, AbstractThread::TailDispatch);
+}
+
+void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream,
+ bool aFinishWhenEnded)
+{
+ MOZ_ASSERT(NS_IsMainThread());
+ DECODER_LOG("AddOutputStream aStream=%p!", aStream);
+ mOutputStreamManager->Add(aStream, aFinishWhenEnded);
+ nsCOMPtr<nsIRunnable> r = NewRunnableMethod<bool>(
+ this, &MediaDecoderStateMachine::SetAudioCaptured, true);
+ OwnerThread()->Dispatch(r.forget());
+}
+
+void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream)
+{
+ MOZ_ASSERT(NS_IsMainThread());
+ DECODER_LOG("RemoveOutputStream=%p!", aStream);
+ mOutputStreamManager->Remove(aStream);
+ if (mOutputStreamManager->IsEmpty()) {
+ nsCOMPtr<nsIRunnable> r = NewRunnableMethod<bool>(
+ this, &MediaDecoderStateMachine::SetAudioCaptured, false);
+ OwnerThread()->Dispatch(r.forget());
+ }
+}
+
+size_t
+MediaDecoderStateMachine::SizeOfVideoQueue() const
+{
+ return mReader->SizeOfVideoQueueInBytes();
+}
+
+size_t
+MediaDecoderStateMachine::SizeOfAudioQueue() const
+{
+ return mReader->SizeOfAudioQueueInBytes();
+}
+
+AbstractCanonical<media::TimeIntervals>*
+MediaDecoderStateMachine::CanonicalBuffered() const
+{
+ return mReader->CanonicalBuffered();
+}
+
+MediaEventSource<void>&
+MediaDecoderStateMachine::OnMediaNotSeekable() const
+{
+ return mReader->OnMediaNotSeekable();
+}
+
+const char*
+MediaDecoderStateMachine::AudioRequestStatus() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mReader->IsRequestingAudioData()) {
+ MOZ_DIAGNOSTIC_ASSERT(!mReader->IsWaitingAudioData());
+ return "pending";
+ } else if (mReader->IsWaitingAudioData()) {
+ return "waiting";
+ }
+ return "idle";
+}
+
+const char*
+MediaDecoderStateMachine::VideoRequestStatus() const
+{
+ MOZ_ASSERT(OnTaskQueue());
+ if (mReader->IsRequestingVideoData()) {
+ MOZ_DIAGNOSTIC_ASSERT(!mReader->IsWaitingVideoData());
+ return "pending";
+ } else if (mReader->IsWaitingVideoData()) {
+ return "waiting";
+ }
+ return "idle";
+}
+
+void
+MediaDecoderStateMachine::OnSuspendTimerResolved()
+{
+ DECODER_LOG("OnSuspendTimerResolved");
+ mVideoDecodeSuspendTimer.CompleteRequest();
+ mStateObj->HandleVideoSuspendTimeout();
+}
+
+void
+MediaDecoderStateMachine::OnSuspendTimerRejected()
+{
+ DECODER_LOG("OnSuspendTimerRejected");
+ MOZ_ASSERT(OnTaskQueue());
+ MOZ_ASSERT(!mVideoDecodeSuspended);
+ mVideoDecodeSuspendTimer.CompleteRequest();
+}
+
+} // namespace mozilla
+
+#undef NS_DispatchToMainThread